From a1c5b1fd9f6ef06a4fa236b7d48350115e5688b9 Mon Sep 17 00:00:00 2001 From: Zhixia Liu Date: Thu, 20 Aug 2026 15:12:43 +0800 Subject: [PATCH] [Core][V1] Support trace_decode_token_ids for deterministic decode replay (#46701) Signed-off-by: ianzxliu Signed-off-by: aoshen02 Signed-off-by: Zhixia Liu Co-authored-by: ianzxliu Co-authored-by: Claude Co-authored-by: aoshen02 Co-authored-by: Codex Co-authored-by: Nick Hill --- docs/serving/online_serving/trace_replay.md | 60 ++++++ examples/generate/trace_replay_offline.py | 195 ++++++++++++++++++ .../test_input_processor_trace_replay.py | 110 ++++++++++ tests/v1/sample/test_trace_replay_params.py | 111 ++++++++++ tests/v1/worker/test_gpu_trace_replay.py | 195 ++++++++++++++++++ vllm/config/model.py | 7 + vllm/config/vllm.py | 8 + vllm/engine/arg_utils.py | 5 + vllm/sampling_params.py | 80 ++++++- vllm/v1/engine/input_processor.py | 41 +++- vllm/v1/worker/gpu/model_runner.py | 1 + vllm/v1/worker/gpu/sample/sampler.py | 14 ++ vllm/v1/worker/gpu/sample/trace_replay.py | 113 ++++++++++ 13 files changed, 931 insertions(+), 9 deletions(-) create mode 100644 docs/serving/online_serving/trace_replay.md create mode 100644 examples/generate/trace_replay_offline.py create mode 100644 tests/v1/engine/test_input_processor_trace_replay.py create mode 100644 tests/v1/sample/test_trace_replay_params.py create mode 100644 tests/v1/worker/test_gpu_trace_replay.py create mode 100644 vllm/v1/worker/gpu/sample/trace_replay.py diff --git a/docs/serving/online_serving/trace_replay.md b/docs/serving/online_serving/trace_replay.md new file mode 100644 index 0000000000..86ed13644c --- /dev/null +++ b/docs/serving/online_serving/trace_replay.md @@ -0,0 +1,60 @@ +# Trace Replay + +Trace replay forces the engine to emit a predetermined token sequence during decoding while computing real logprobs from the model's unmodified logit distribution. The primary use case is comparing logprob distributions across different configurations: + +- **Inference config diff**: compare logprobs between different quantization schemes, tensor parallelism layouts, or attention backends for the same model. +- **Train vs inference diff**: replay a training-time token sequence through the inference engine to detect logprob divergence caused by numerical differences between training and serving frameworks. + +## Requirements + +Trace replay reserves a per-request trace buffer, so it is off by default. Enable it with `--enable-trace-replay` (or `LLM(..., enable_trace_replay=True)`). It is only supported by model runner V2; requests are rejected when either condition is not met. + +## Usage + +```python +from vllm import LLM, SamplingParams + +llm = LLM(model="Qwen/Qwen3-0.6B", enable_trace_replay=True) + +# Token sequence captured from a previous run or training log +trace_tokens = [15, 284, 1026, 374] + +params = SamplingParams( + trace_decode_token_ids=trace_tokens, + logprobs=5, +) +outputs = llm.generate(["Once upon a time"], sampling_params=params) + +for token, logprob in zip( + outputs[0].outputs[0].token_ids, + outputs[0].outputs[0].logprobs, +): + print(f"token={token} logprob={logprob[token].logprob:.4f}") +``` + +The output tokens will always be `[15, 284, 1026, 374]`. The logprobs reflect the model's true probability for each forced token under the current inference configuration. + +To compare configurations, run the same trace against two engine setups and diff the per-token logprobs. + +## Behavior + +When `trace_decode_token_ids` is set: + +- `max_tokens` is automatically set to the trace length. +- The trace is truncated if it does not fit within `max_model_len` given the prompt length. +- All stop conditions (EOS, stop strings, stop token IDs) are disabled. +- Generation produces exactly the trace tokens, then stops. + +## Limitations + +`trace_decode_token_ids` is incompatible with the following features (raises `ValueError`): + +| Feature | Reason | +| --- | --- | +| `n > 1` | Trace replay produces a single deterministic sequence | +| `prompt_logprobs` | Expanded logit layout conflicts with trace kernel indexing | +| Speculative decoding | Multi-token speculation conflicts with single-token trace stepping | +| Structured outputs | Grammar constraints conflict with forced token injection | +| `repetition_detection` | Would terminate the request on repeated patterns in the trace | +| `thinking_token_budget` | Logit masking corrupts logprobs for trace tokens | +| `bad_words` | Logit masking corrupts logprobs for trace tokens | diff --git a/examples/generate/trace_replay_offline.py b/examples/generate/trace_replay_offline.py new file mode 100644 index 0000000000..2baa03307d --- /dev/null +++ b/examples/generate/trace_replay_offline.py @@ -0,0 +1,195 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Trace-replay with vLLM offline inference. + +Trace-replay lets you supply a known sequence of decode token IDs alongside +the prompt. Instead of sampling from the model distribution, the engine +injects each decode token deterministically, step by step. All other +outputs — logprobs, token ranks, text decoding — are computed faithfully +from the real logit distribution for that token. + +How it works: + 1. Start the engine with ``--enable-trace-replay`` (or + ``LLM(..., enable_trace_replay=True)``). + 2. Set ``SamplingParams.trace_decode_token_ids`` to the list of decode + token IDs you want to force. + 3. The engine will output exactly those tokens and stop. ``max_tokens`` + is overwritten with the trace length, and the trace is truncated if it + does not fit within ``max_model_len``. EOS tokens inside the trace + sequence do **not** halt generation early. + +Requires model runner V2. Requests are rejected with ``ValueError`` when +combined with any of: + * n > 1 * Speculative decoding + * prompt_logprobs * Structured outputs + * repetition_detection * thinking_token_budget + * bad_words + +Typical use-cases: + * Reproduce exact outputs from a previous run for benchmarking. + * Compute logprobs for an already-known output (e.g. reference answers). + * Dataset annotation: given (prompt, response) pairs, obtain per-token + logprob scores without altering the response. + +Usage: + python examples/generate/trace_replay_offline.py + python examples/generate/trace_replay_offline.py --model facebook/opt-125m +""" + +import argparse + +from vllm import LLM, SamplingParams + +DEFAULT_PROMPT = "Hello, my name is" + + +def build_llm(args: argparse.Namespace) -> LLM: + """Construct an LLM from common CLI args.""" + llm_kwargs: dict = { + "model": args.model, + "trust_remote_code": args.trust_remote_code, + "tensor_parallel_size": args.tensor_parallel_size, + "enforce_eager": args.enforce_eager, + "gpu_memory_utilization": args.gpu_memory_utilization, + "max_num_seqs": args.max_num_seqs, + "enable_trace_replay": True, + } + if args.max_model_len is not None: + llm_kwargs["max_model_len"] = args.max_model_len + return LLM(**llm_kwargs) + + +def run_normal_generation(llm: LLM, prompt: str, max_tokens: int) -> list[int]: + """Run a standard greedy generation and return the output token IDs.""" + sampling_params = SamplingParams( + temperature=0.0, + max_tokens=max_tokens, + # Request logprobs for the greedy token at each step so we can + # compare them against the trace-replay logprobs below. + logprobs=1, + ) + outputs = llm.generate([prompt], sampling_params=sampling_params) + result = outputs[0].outputs[0] + output_token_ids = list(result.token_ids) + print("[Normal generation]") + print(f" Prompt : {prompt!r}") + print(f" Output token IDs : {output_token_ids}") + print(f" Output text : {result.text!r}") + return output_token_ids + + +def run_trace_replay(llm: LLM, prompt: str, decode_token_ids: list[int]) -> None: + """Replay a known decode sequence and print per-token logprobs.""" + sampling_params = SamplingParams( + # Provide the decode tokens to replay. + trace_decode_token_ids=decode_token_ids, + # Request top-5 logprobs so we can inspect the distribution. + logprobs=5, + ) + outputs = llm.generate([prompt], sampling_params=sampling_params) + result = outputs[0].outputs[0] + replayed_ids = list(result.token_ids) + + print("\n[Trace-replay]") + print(f" Requested decode token IDs : {decode_token_ids}") + print(f" Replayed output token IDs : {replayed_ids}") + print(f" Replayed output text : {result.text!r}") + + # Verify the replayed tokens match exactly. + assert replayed_ids == decode_token_ids, ( + f"Mismatch!\n expected: {decode_token_ids}\n got: {replayed_ids}" + ) + print(" Replayed tokens match the requested trace exactly.") + + # Show per-token logprobs (computed from the real distribution). + if result.logprobs: + print("\n Per-token logprobs (trace token):") + for step, (token_id, logprob_dict) in enumerate( + zip(replayed_ids, result.logprobs) + ): + sampled_lp = logprob_dict.get(token_id) + lp_value = f"{sampled_lp.logprob:.4f}" if sampled_lp is not None else "n/a" + rank = sampled_lp.rank if sampled_lp is not None else "n/a" + print( + f" step {step:2d}: token_id={token_id:6d} " + f"logprob={lp_value} rank={rank}" + ) + + +def run_demo(args: argparse.Namespace) -> None: + print(f"Loading model: {args.model}\n") + llm = build_llm(args) + prompt = args.prompt + + print("=" * 60) + print("Step 1 — Normal greedy generation (captures decode tokens)") + print("=" * 60) + decode_token_ids = run_normal_generation(llm, prompt, max_tokens=8) + + print("\n" + "=" * 60) + print("Step 2 — Trace-replay with the captured tokens") + print("=" * 60) + run_trace_replay(llm, prompt, decode_token_ids) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Trace-replay with vLLM offline inference" + ) + parser.add_argument( + "--prompt", + type=str, + default=DEFAULT_PROMPT, + help="Text prompt to use for generation (default: %(default)r)", + ) + parser.add_argument( + "--model", + type=str, + default="facebook/opt-125m", + help="Name or path of the HuggingFace model to use", + ) + parser.add_argument( + "--trust-remote-code", + action="store_true", + help="Trust remote code from HuggingFace", + ) + parser.add_argument( + "--tensor-parallel-size", + type=int, + default=1, + help="Number of tensor parallel replicas", + ) + parser.add_argument( + "--enforce-eager", + action="store_true", + help="Always use eager-mode PyTorch (disable CUDA graph)", + ) + parser.add_argument( + "--gpu-memory-utilization", + type=float, + default=0.9, + help="Fraction of GPU memory to use", + ) + parser.add_argument( + "--max-model-len", + type=int, + default=None, + help="Model context length", + ) + parser.add_argument( + "--max-num-seqs", + type=int, + default=256, + help="Maximum number of sequences per iteration", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + run_demo(args) + + +if __name__ == "__main__": + main() diff --git a/tests/v1/engine/test_input_processor_trace_replay.py b/tests/v1/engine/test_input_processor_trace_replay.py new file mode 100644 index 0000000000..894293bc3d --- /dev/null +++ b/tests/v1/engine/test_input_processor_trace_replay.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import time +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from vllm import SamplingParams +from vllm.config import VllmConfig +from vllm.exceptions import VLLMValidationError +from vllm.v1.core.sched.utils import check_stop +from vllm.v1.engine.input_processor import InputProcessor +from vllm.v1.request import Request, RequestStatus + +pytestmark = pytest.mark.skip_global_cleanup + + +def _make_request(params: SamplingParams) -> Request: + return Request( + request_id="req-0", + client_index=0, + prompt_token_ids=[1, 2, 3], + sampling_params=params, + pooling_params=None, + arrival_time=time.time(), + ) + + +def _normalize(params: SamplingParams, prompt_len: int, max_model_len: int) -> None: + processor = SimpleNamespace( + model_config=SimpleNamespace(max_model_len=max_model_len) + ) + InputProcessor._normalize_trace_replay_params(processor, params, prompt_len) + + +def test_normalize_trace_replay_params(): + params = SamplingParams( + max_tokens=16, + min_tokens=10, + stop_token_ids=[20], + trace_decode_token_ids=[10, 20, 30], + ) + params.update_from_generation_config({}, eos_token_id=20) + + _normalize(params, prompt_len=3, max_model_len=128) + + assert params.max_tokens == 3 + assert params.min_tokens == 0 + assert params.ignore_eos is True + assert params.eos_token_id is None + assert params.stop_token_ids == [] + assert params.all_stop_token_ids == set() + + request = _make_request(params) + assert request.max_tokens == 3 + request.append_output_token_ids([10, 20, 30]) + assert check_stop(request, max_model_len=128) + assert request.status == RequestStatus.FINISHED_LENGTH_CAPPED + + +def test_trace_longer_than_remaining_context_is_truncated(): + """The trace is staged into a max_model_len-wide row, so it must be cut. + + An explicitly set max_tokens is never clamped to max_model_len, so it cannot + bound the staged write. + """ + params = SamplingParams(max_tokens=100, trace_decode_token_ids=list(range(20))) + + _normalize(params, prompt_len=6, max_model_len=10) + + assert params.trace_decode_token_ids == [0, 1, 2, 3] + assert params.max_tokens == 4 + + +def _validate(enable_trace_replay: bool) -> None: + """Run _validate_params' trace gating with the rest of verify() stubbed.""" + processor = SimpleNamespace( + model_config=SimpleNamespace( + return_sampling_mask=False, + enable_trace_replay=enable_trace_replay, + ), + vllm_config=SimpleNamespace(reasoning_config=None), + speculative_config=None, + structured_outputs_config=None, + tokenizer=None, + ) + params = SamplingParams(trace_decode_token_ids=[1, 2, 3]) + with patch.object(SamplingParams, "verify"): + InputProcessor._validate_params(processor, params, ("generate",)) + + +def test_trace_request_rejected_when_feature_disabled(): + with pytest.raises(VLLMValidationError, match="--enable-trace-replay"): + _validate(enable_trace_replay=False) + + +def test_trace_request_accepted_when_feature_enabled(): + _validate(enable_trace_replay=True) + + +def test_trace_replay_requires_v2_model_runner(): + config = SimpleNamespace( + model_config=SimpleNamespace(enable_trace_replay=True), + use_v2_model_runner=False, + ) + + with pytest.raises(ValueError, match="trace replay requires Model Runner V2"): + VllmConfig._verify_trace_replay_config(config) diff --git a/tests/v1/sample/test_trace_replay_params.py b/tests/v1/sample/test_trace_replay_params.py new file mode 100644 index 0000000000..6e0d726470 --- /dev/null +++ b/tests/v1/sample/test_trace_replay_params.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from vllm import SamplingParams +from vllm.exceptions import VLLMValidationError +from vllm.sampling_params import StructuredOutputsParams + +pytestmark = pytest.mark.skip_global_cleanup + +# --------------------------------------------------------------------------- +# SamplingParams field +# --------------------------------------------------------------------------- + + +def test_sampling_params_trace_field_defaults_to_none(): + params = SamplingParams(max_tokens=10) + assert params.trace_decode_token_ids is None + + +def test_sampling_params_trace_field_accepts_list(): + ids = [100, 200, 300] + params = SamplingParams(trace_decode_token_ids=ids) + assert params.trace_decode_token_ids == ids + + +def test_sampling_params_trace_field_preserved_by_clone(): + ids = [1, 2, 3] + params = SamplingParams(trace_decode_token_ids=ids) + cloned = params.clone() + assert cloned.trace_decode_token_ids == ids + assert cloned.trace_decode_token_ids is not params.trace_decode_token_ids + + +def test_sampling_params_trace_field_rejects_empty_list(): + params = SamplingParams(trace_decode_token_ids=[]) + with pytest.raises(ValueError, match="non-empty"): + params._validate_trace_replay( + _make_model_config(vocab_size=100), speculative_config=None + ) + + +def test_sampling_params_trace_field_requires_single_output(): + params = SamplingParams(n=2, trace_decode_token_ids=[1]) + with pytest.raises(ValueError, match="requires n=1"): + params._validate_trace_replay( + _make_model_config(vocab_size=100), speculative_config=None + ) + + +@pytest.mark.parametrize("invalid_ids", [[-1, 5], [1, "2"]]) +def test_sampling_params_trace_field_rejects_invalid_token_ids(invalid_ids): + params = SamplingParams(trace_decode_token_ids=invalid_ids) + with pytest.raises(ValueError, match="non-negative integers"): + params._validate_trace_replay( + _make_model_config(vocab_size=100), speculative_config=None + ) + + +def _make_model_config(vocab_size: int): + from unittest.mock import Mock + + model_config = Mock() + model_config.get_vocab_size = lambda: vocab_size + return model_config + + +def test_validate_trace_replay_accepts_in_vocab(): + params = SamplingParams(trace_decode_token_ids=[0, 50, 99]) + # Should not raise. + params._validate_trace_replay( + _make_model_config(vocab_size=100), speculative_config=None + ) + + +def test_validate_trace_replay_rejects_out_of_vocab(): + # The non-negative check passes at construction, but the token id exceeds + # the vocabulary; verify() must reject it before it reaches the sampler. + params = SamplingParams(trace_decode_token_ids=[0, 100]) + with pytest.raises(VLLMValidationError, match="out-of-vocab"): + params._validate_trace_replay( + _make_model_config(vocab_size=100), speculative_config=None + ) + + +def test_validate_trace_replay_noop_when_unset(): + params = SamplingParams(max_tokens=4) + # Should not raise when the field is unset. + params._validate_trace_replay( + _make_model_config(vocab_size=100), speculative_config=None + ) + + +def test_trace_decode_token_ids_rejects_speculative_decoding(): + params = SamplingParams(trace_decode_token_ids=[1]) + with pytest.raises(ValueError, match="not supported with speculative decoding"): + params._validate_trace_replay( + _make_model_config(vocab_size=100), speculative_config=object() + ) + + +def test_trace_decode_token_ids_rejects_structured_outputs(): + params = SamplingParams( + trace_decode_token_ids=[1], + structured_outputs=StructuredOutputsParams(json_object=True), + ) + with pytest.raises(ValueError, match="not supported with structured outputs"): + params._validate_trace_replay( + _make_model_config(vocab_size=100), speculative_config=None + ) diff --git a/tests/v1/worker/test_gpu_trace_replay.py b/tests/v1/worker/test_gpu_trace_replay.py new file mode 100644 index 0000000000..9d418101fc --- /dev/null +++ b/tests/v1/worker/test_gpu_trace_replay.py @@ -0,0 +1,195 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for Model Runner V2 inference trace-replay. + +Trace-replay forces the sampler to emit a predetermined sequence of decode +token IDs. The replay step for a request is derived from GPU state as +``total_len - prompt_len`` (output tokens produced so far), and the trace token +overwrites the sampled token in place before logprobs are computed. +""" + +from types import SimpleNamespace +from typing import cast + +import pytest +import torch + +pytest.importorskip("triton") +if not torch.cuda.is_available(): + pytest.skip("CUDA required for trace-replay tests", allow_module_level=True) + +from vllm.sampling_params import SamplingParams +from vllm.v1.worker.gpu.sample.trace_replay import ( + TraceReplayState, + apply_trace_tokens, +) +from vllm.v1.worker.gpu.states import RequestState + +DEVICE = "cuda" +TEST_MAX_MODEL_LEN = 2048 + + +def _i32(x) -> torch.Tensor: + return torch.tensor(x, dtype=torch.int32, device=DEVICE) + + +def _i64(x) -> torch.Tensor: + return torch.tensor(x, dtype=torch.int64, device=DEVICE) + + +def _trace_state(max_num_reqs: int) -> TraceReplayState: + """Build a state whose req_states exposes the buffers apply_trace reads. + + total_len/prompt_len are written by the caller to position each request at + the desired replay step. + """ + req_states = cast( + RequestState, + SimpleNamespace( + max_num_reqs=max_num_reqs, + max_model_len=TEST_MAX_MODEL_LEN, + device=torch.device(DEVICE), + total_len=SimpleNamespace( + gpu=torch.zeros(max_num_reqs, dtype=torch.int32, device=DEVICE) + ), + prompt_len=SimpleNamespace( + gpu=torch.zeros(max_num_reqs, dtype=torch.int32, device=DEVICE) + ), + ), + ) + return TraceReplayState(req_states) + + +def _set_lens(state: TraceReplayState, total_len, prompt_len) -> None: + """Position each request at a replay step via total_len - prompt_len.""" + state.req_states.total_len.gpu[: len(total_len)] = _i32(total_len) + state.req_states.prompt_len.gpu[: len(prompt_len)] = _i32(prompt_len) + + +# ------------------------------ Kernel ------------------------------------ + + +def test_replay_overwrites_sampled_at_each_step(): + """The trace token for the current step replaces the sampled token.""" + trace = [[100, 101, 102], [200, 201, 202]] + trace_len = _i32([3, 3]) + trace_token_ids = torch.zeros( + 2, TEST_MAX_MODEL_LEN, dtype=torch.int32, device=DEVICE + ) + for i, t in enumerate(trace): + trace_token_ids[i, : len(t)] = _i32(t) + prompt_len = _i32([5, 8]) + idx_mapping = _i32([0, 1]) + + for step in range(3): + sampled = _i64([-7, -7]) # sentinel that must be overwritten + total_len = _i32([5 + step, 8 + step]) + apply_trace_tokens( + sampled, idx_mapping, trace_token_ids, trace_len, total_len, prompt_len + ) + assert sampled.tolist() == [trace[0][step], trace[1][step]] + + +def test_past_end_of_trace_leaves_sampled_untouched(): + """Once step >= trace_len, the sampler's own token is kept.""" + trace_token_ids = torch.zeros( + 1, TEST_MAX_MODEL_LEN, dtype=torch.int32, device=DEVICE + ) + trace_token_ids[0, :2] = _i32([100, 101]) + trace_len = _i32([2]) + prompt_len = _i32([4]) + idx_mapping = _i32([0]) + + sampled = _i64([999]) + total_len = _i32([4 + 2]) # step == 2 == trace_len -> out of range + apply_trace_tokens( + sampled, idx_mapping, trace_token_ids, trace_len, total_len, prompt_len + ) + assert sampled.tolist() == [999] + + +def test_non_trace_request_untouched(): + """trace_len == 0 means the request never uses replay.""" + trace_token_ids = torch.zeros( + 1, TEST_MAX_MODEL_LEN, dtype=torch.int32, device=DEVICE + ) + trace_len = _i32([0]) + prompt_len = _i32([3]) + idx_mapping = _i32([0]) + + sampled = _i64([42]) + total_len = _i32([3]) + apply_trace_tokens( + sampled, idx_mapping, trace_token_ids, trace_len, total_len, prompt_len + ) + assert sampled.tolist() == [42] + + +def test_idx_mapping_indirection_and_negative_skip(): + """batch_idx -> req_state_idx indirection, and negative entries are skipped.""" + # req_state 0: no trace; req_state 1: trace [500, 501]. + trace_token_ids = torch.zeros( + 2, TEST_MAX_MODEL_LEN, dtype=torch.int32, device=DEVICE + ) + trace_token_ids[1, :2] = _i32([500, 501]) + trace_len = _i32([0, 2]) + prompt_len = _i32([10, 6]) + + # batch: [maps to state 1, masked (-1), maps to state 0]. + idx_mapping = _i32([1, -1, 0]) + sampled = _i64([1, 2, 3]) + # Indexed by req_state_idx: state 0 at total_len=10, state 1 at step 1. + total_len = _i32([10, 6 + 1]) + apply_trace_tokens( + sampled, idx_mapping, trace_token_ids, trace_len, total_len, prompt_len + ) + # batch 0 -> state 1 step 1 -> 501; batch 1 masked; batch 2 -> state 0 no trace. + assert sampled.tolist() == [501, 2, 3] + + +# --------------------------- TraceReplayState ----------------------------- + + +def test_state_end_to_end(): + """add_request -> apply_staged_writes -> apply_trace overwrites correctly.""" + state = _trace_state(4) + state.add_request(0, SamplingParams(trace_decode_token_ids=[11, 22, 33])) + state.add_request(1, SamplingParams()) # no trace + state.apply_staged_writes() + + idx_mapping = _i32([0, 1]) + sampled = _i64([-1, -1]) + _set_lens(state, [7 + 1, 7, 0, 0], [7, 7, 0, 0]) # req 0 at step 1 + state.apply_trace(sampled, idx_mapping) + assert sampled.tolist() == [22, -1] + + +def test_state_leaves_non_trace_batch_unchanged(): + """Requests with trace_len == 0 remain unchanged after a trace was seen.""" + state = _trace_state(4) + state.add_request(0, SamplingParams(trace_decode_token_ids=[11, 22])) + state.add_request(1, SamplingParams()) + state.apply_staged_writes() + + # Only the non-trace request (state 1) is in this batch. + idx_mapping = _i32([1]) + sampled = _i64([555]) + _set_lens(state, [0, 7, 0, 0], [7, 7, 0, 0]) + state.apply_trace(sampled, idx_mapping) + assert sampled.tolist() == [555] + + +def test_slot_reuse_clears_trace(): + """Reusing a slot for a non-trace request must not replay stale tokens.""" + state = _trace_state(2) + state.add_request(0, SamplingParams(trace_decode_token_ids=[11, 22])) + state.apply_staged_writes() + # Slot 0 reused by a request without a trace. + state.add_request(0, SamplingParams()) + state.apply_staged_writes() + + idx_mapping = _i32([0]) + sampled = _i64([888]) + _set_lens(state, [3, 0], [3, 0]) + state.apply_trace(sampled, idx_mapping) + assert sampled.tolist() == [888] diff --git a/vllm/config/model.py b/vllm/config/model.py index 16ff966c2d..56e533ea53 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -262,6 +262,12 @@ class ModelConfig: equivalent exponential-race sampling. FP64 preserves lower-tail sampling events that fp32 uniform/exponential draws can truncate, at the cost of significantly lower throughput on most GPUs.""" + enable_trace_replay: bool = False + """Whether to allow requests to set + `SamplingParams.trace_decode_token_ids`, which forces decoding to follow a + predetermined token sequence while still computing real logprobs. Reserved + for debugging and RL workflows: enabling it reserves a per-request trace + buffer, so it is off by default.""" disable_sliding_window: bool = False """Whether to disable sliding window. If True, we will disable the sliding window functionality of the model, capping to sliding window size. If the @@ -424,6 +430,7 @@ class ModelConfig: "return_sampling_mask", "logprobs_mode", "use_fp64_gumbel", + "enable_trace_replay", "disable_cascade_attn", "skip_tokenizer_init", "served_model_name", diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index ade8a666cf..a271a65413 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1091,6 +1091,13 @@ class VllmConfig: "are normalized over the same nucleus as the sampling mask" ) + def _verify_trace_replay_config(self) -> None: + model_config = self.model_config + if model_config is None or not model_config.enable_trace_replay: + return + if not self.use_v2_model_runner: + raise ValueError("trace replay requires Model Runner V2") + def __post_init__(self): """Verify configs are valid & consistent with each other.""" @@ -1141,6 +1148,7 @@ class VllmConfig: ) self._verify_sampling_replay_config() + self._verify_trace_replay_config() if self.lora_config is not None: self.lora_config.verify_with_model_config(self.model_config) diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 1fb1388f10..035595d5a8 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -543,6 +543,7 @@ class EngineArgs: max_logprobs: int = ModelConfig.max_logprobs logprobs_mode: LogprobsMode = ModelConfig.logprobs_mode use_fp64_gumbel: bool = ModelConfig.use_fp64_gumbel + enable_trace_replay: bool = ModelConfig.enable_trace_replay disable_log_stats: bool = False aggregate_engine_logging: bool = False revision: str | None = ModelConfig.revision @@ -884,6 +885,9 @@ class EngineArgs: model_group.add_argument("--max-logprobs", **model_kwargs["max_logprobs"]) model_group.add_argument("--logprobs-mode", **model_kwargs["logprobs_mode"]) model_group.add_argument("--use-fp64-gumbel", **model_kwargs["use_fp64_gumbel"]) + model_group.add_argument( + "--enable-trace-replay", **model_kwargs["enable_trace_replay"] + ) model_group.add_argument( "--disable-sliding-window", **model_kwargs["disable_sliding_window"] ) @@ -1760,6 +1764,7 @@ class EngineArgs: max_logprobs=self.max_logprobs, logprobs_mode=self.logprobs_mode, use_fp64_gumbel=self.use_fp64_gumbel, + enable_trace_replay=self.enable_trace_replay, disable_sliding_window=self.disable_sliding_window, disable_cascade_attn=self.disable_cascade_attn, skip_tokenizer_init=self.skip_tokenizer_init, diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 2f0af9e1b4..f0e325c255 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -346,14 +346,6 @@ class SamplingParams( """Arbitrary additional args, that can be used by custom sampling implementations, plugins, etc. Not used by any in-tree sampling implementations.""" - routed_experts_prompt_start: int = 0 - """When enable_return_routed_experts is active, skip the first - routed_experts_prompt_start prompt tokens from the returned routing - data. In multi-turn agent scenarios, set this to the length of the - already-returned prefix to avoid duplicating routing for prompt tokens - covered by earlier turns. Default 0 returns routing for all prompt - tokens.""" - # Fields used for bad words bad_words: list[str] | None = None """Words that are not allowed to be generated. More precisely, only the @@ -373,6 +365,19 @@ class SamplingParams( '\\emoji \\emoji \\emoji ...'). This feature can detect such behavior and terminate early, saving time and tokens.""" + # Debugging / RL-specific parameters. Not intended for production serving. + routed_experts_prompt_start: int = 0 + """When enable_return_routed_experts is active, skip the first + routed_experts_prompt_start prompt tokens from the returned routing + data. In multi-turn agent scenarios, set this to the length of the + already-returned prefix to avoid duplicating routing for prompt tokens + covered by earlier turns. Default 0 returns routing for all prompt + tokens.""" + trace_decode_token_ids: list[int] | None = None + """If provided, forces the engine to emit this predetermined sequence of + token IDs during decoding instead of sampling randomly. Real logprobs are + still computed. Conflict checking is performed at the engine level.""" + @staticmethod def from_optional( n: int | None = 1, @@ -407,6 +412,8 @@ class SamplingParams( repetition_detection: RepetitionDetectionParams | None = None, logprob_token_ids: list[int] | None = None, routed_experts_prompt_start: int = 0, + # Debugging / RL-specific parameters. + trace_decode_token_ids: list[int] | None = None, ) -> "SamplingParams": if logit_bias is not None: # Fast path uses a dict comprehension; on failure we iterate once @@ -470,6 +477,7 @@ class SamplingParams( skip_clone=skip_clone, repetition_detection=repetition_detection, routed_experts_prompt_start=routed_experts_prompt_start, + trace_decode_token_ids=trace_decode_token_ids, ) def __post_init__(self) -> None: @@ -780,6 +788,7 @@ class SamplingParams( ) -> None: self._validate_logprobs(model_config) self._validate_logit_bias(model_config) + self._validate_trace_replay(model_config, speculative_config) self._validate_logits_processors(model_config) self._validate_allowed_token_ids(tokenizer) self._validate_spec_decode(speculative_config) @@ -870,6 +879,61 @@ class SamplingParams( value=invalid_token_ids, ) + def _validate_trace_replay( + self, + model_config: ModelConfig, + speculative_config: SpeculativeConfig | None, + ) -> None: + """Validate trace replay request compatibility.""" + if self.trace_decode_token_ids is None: + return + + if len(self.trace_decode_token_ids) == 0: + raise ValueError("trace_decode_token_ids must be a non-empty list.") + if self.n != 1: + raise ValueError("trace_decode_token_ids requires n=1.") + if not all(isinstance(t, int) and t >= 0 for t in self.trace_decode_token_ids): + raise ValueError( + "trace_decode_token_ids must contain non-negative integers." + ) + + if self.prompt_logprobs is not None: + raise ValueError( + "trace_decode_token_ids is not supported with prompt_logprobs." + ) + if speculative_config is not None: + raise ValueError( + "trace_decode_token_ids is not supported with speculative decoding." + ) + if self.structured_outputs is not None: + raise ValueError( + "trace_decode_token_ids is not supported with structured outputs." + ) + if self.repetition_detection is not None: + raise ValueError( + "trace_decode_token_ids is not supported with repetition_detection." + ) + if self.thinking_token_budget is not None: + raise ValueError( + "trace_decode_token_ids is not supported with thinking_token_budget." + ) + if self.bad_words: + raise ValueError("trace_decode_token_ids is not supported with bad_words.") + + vocab_size = model_config.get_vocab_size() + invalid_token_ids = [ + token_id + for token_id in self.trace_decode_token_ids + if token_id < 0 or token_id >= vocab_size + ] + if invalid_token_ids: + raise VLLMValidationError( + f"token_id(s) {invalid_token_ids} in trace_decode_token_ids " + f"contain out-of-vocab token ids. Vocabulary size: {vocab_size}", + parameter="trace_decode_token_ids", + value=invalid_token_ids, + ) + def _validate_logits_processors(self, model_config: ModelConfig) -> None: from vllm.v1.sample.logits_processor import ( validate_logits_processors_parameters, diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index ac870c931a..744da670a0 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -51,7 +51,6 @@ class InputProcessor: self.speculative_config = vllm_config.speculative_config self.structured_outputs_config = vllm_config.structured_outputs_config self.observability_config = vllm_config.observability_config - self.use_v2_model_runner = vllm_config.use_v2_model_runner self.generation_config_fields = model_config.try_get_generation_config() @@ -128,6 +127,15 @@ class InputProcessor: "not configured. Please set --reasoning-parser " "and/or --reasoning-config to use thinking_token_budget." ) + if ( + params.trace_decode_token_ids + and not self.model_config.enable_trace_replay + ): + raise VLLMValidationError( + "trace_decode_token_ids is set but trace replay is not " + "enabled. Start the engine with --enable-trace-replay " + "to use it." + ) elif isinstance(params, PoolingParams): supported_pooling_tasks = [ task for task in supported_tasks if task in POOLING_TASKS @@ -156,6 +164,30 @@ class InputProcessor: f"but got {type(params).__name__}" ) + def _normalize_trace_replay_params( + self, sampling_params: SamplingParams, prompt_len: int + ) -> None: + """Apply trace replay's generation semantics to request-local params.""" + trace_token_ids = sampling_params.trace_decode_token_ids + assert trace_token_ids + assert sampling_params.max_tokens is not None + + max_trace_len = max(self.model_config.max_model_len - prompt_len, 1) + trace_token_ids = trace_token_ids[:max_trace_len] + sampling_params.trace_decode_token_ids = trace_token_ids + + # Apply this after the generation config so its EOS token cannot stop + # replay before the trace is exhausted. + sampling_params.max_tokens = min( + len(trace_token_ids), sampling_params.max_tokens + ) + sampling_params.min_tokens = 0 + sampling_params.ignore_eos = True + sampling_params._eos_token_id = None + sampling_params.stop = [] + sampling_params.stop_token_ids = [] + sampling_params._all_stop_token_ids = set() + def _validate_lora(self, lora_request: LoRARequest | None) -> None: if lora_request is None: return @@ -340,6 +372,13 @@ class InputProcessor: ) if self.tokenizer is not None: sampling_params.update_from_tokenizer(self.tokenizer) + if sampling_params.trace_decode_token_ids: + self._normalize_trace_replay_params( + sampling_params, + length_from_prompt_token_ids_or_embeds( + prompt_token_ids, prompt_embeds + ), + ) else: pooling_params = params.clone() diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 3861bb5264..66b3c8db4a 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -408,6 +408,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): logprobs_mode=self.model_config.logprobs_mode, num_speculative_tokens=self.decode_query_len, use_fp64_gumbel=self.model_config.use_fp64_gumbel, + enable_trace_replay=self.model_config.enable_trace_replay, reasoning_config=self.vllm_config.reasoning_config, return_sampling_mask=self.model_config.return_sampling_mask, ) diff --git a/vllm/v1/worker/gpu/sample/sampler.py b/vllm/v1/worker/gpu/sample/sampler.py index 26371f014e..9e65d9a304 100644 --- a/vllm/v1/worker/gpu/sample/sampler.py +++ b/vllm/v1/worker/gpu/sample/sampler.py @@ -26,6 +26,7 @@ from vllm.v1.worker.gpu.sample.output import SamplerOutput, SamplingMaskTensors from vllm.v1.worker.gpu.sample.penalties import PenaltiesState from vllm.v1.worker.gpu.sample.states import NO_LOGPROBS, SamplingStates from vllm.v1.worker.gpu.sample.thinking_budget import ThinkingBudgetState +from vllm.v1.worker.gpu.sample.trace_replay import TraceReplayState from vllm.v1.worker.gpu.states import RequestState @@ -39,6 +40,7 @@ class Sampler: logprobs_mode: LogprobsMode = "raw_logprobs", num_speculative_tokens: int = 1, use_fp64_gumbel: bool = False, + enable_trace_replay: bool = False, reasoning_config: ReasoningConfig | None = None, return_sampling_mask: bool = False, ): @@ -53,6 +55,9 @@ class Sampler: self.bad_words_state = BadWordsState(req_states) self.logprob_token_ids_state = LogprobTokenIdsState(max_num_reqs, device) self.thinking_budget_state = ThinkingBudgetState(req_states, reasoning_config) + self.trace_replay_state = ( + TraceReplayState(req_states) if enable_trace_replay else None + ) self.needs_logits_processing = np.zeros(max_num_reqs, dtype=bool) self.num_speculative_tokens = num_speculative_tokens self.return_sampling_mask = return_sampling_mask @@ -69,6 +74,8 @@ class Sampler: self.bad_words_state.add_request(req_idx, sampling_params) self.logprob_token_ids_state.add_request(req_idx, sampling_params) self.thinking_budget_state.add_request(req_idx, sampling_params) + if self.trace_replay_state is not None: + self.trace_replay_state.add_request(req_idx, sampling_params) states = self.sampling_states temperature = states.temperature.np[req_idx] @@ -93,6 +100,8 @@ class Sampler: self.bad_words_state.apply_staged_writes() self.logprob_token_ids_state.apply_staged_writes() self.thinking_budget_state.apply_staged_writes() + if self.trace_replay_state is not None: + self.trace_replay_state.apply_staged_writes() def __call__( self, @@ -128,6 +137,11 @@ class Sampler: return_logprobs=return_logprobs, ) + if self.trace_replay_state is not None: + # Overwrite sampled tokens with the replay trace up-front so that + # computed logprobs reflect the real distribution of the forced token. + self.trace_replay_state.apply_trace(sampled, idx_mapping) + if return_logprobs: if self.logprobs_mode in PROCESSED_LOGPROBS_MODES: logits = processed_logits diff --git a/vllm/v1/worker/gpu/sample/trace_replay.py b/vllm/v1/worker/gpu/sample/trace_replay.py new file mode 100644 index 0000000000..9b58d4cf29 --- /dev/null +++ b/vllm/v1/worker/gpu/sample/trace_replay.py @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import torch + +from vllm.sampling_params import SamplingParams +from vllm.triton_utils import tl, triton +from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor +from vllm.v1.worker.gpu.states import RequestState + + +class TraceReplayState: + """Per-request state for inference trace-replay. + + When a request carries ``SamplingParams.trace_decode_token_ids``, the + sampler overwrites the sampled token at each decode step with the + predetermined trace token, while real logprobs and ranks are still computed + from the unmodified logit distribution. The replay step for a request is + derived entirely from GPU state (``total_len - prompt_len``), so no CPU + synchronization or async placeholder handling is needed. + """ + + def __init__(self, req_states: RequestState): + self.max_num_reqs = req_states.max_num_reqs + self.device = req_states.device + self.req_states = req_states + self.trace_token_ids = StagedWriteTensor( + (self.max_num_reqs, req_states.max_model_len), + dtype=torch.int32, + device=self.device, + uva_instead_of_gpu=True, + ) + self.trace_len = UvaBackedTensor(self.max_num_reqs, dtype=torch.int32) + + def add_request(self, req_idx: int, sampling_params: SamplingParams) -> None: + trace = sampling_params.trace_decode_token_ids + if trace is not None: + self.trace_len.np[req_idx] = len(trace) + self.trace_token_ids.stage_write(req_idx, 0, trace) + else: + self.trace_len.np[req_idx] = 0 + + def apply_staged_writes(self) -> None: + self.trace_len.copy_to_uva() + self.trace_token_ids.apply_write() + + def apply_trace( + self, + sampled: torch.Tensor, + idx_mapping: torch.Tensor, + ) -> None: + apply_trace_tokens( + sampled, + idx_mapping, + self.trace_token_ids.gpu, + self.trace_len.gpu, + self.req_states.total_len.gpu, + self.req_states.prompt_len.gpu, + ) + + +@triton.jit +def _trace_replay_kernel( + sampled_ptr, # [num_reqs], int64, mutated in place + idx_mapping_ptr, # [num_reqs] batch_idx -> req_state_idx + trace_token_ids_ptr, # [max_num_reqs, max_model_len], int32 + trace_token_ids_stride, + trace_len_ptr, # [max_num_reqs], int32 + total_len_ptr, # [max_num_reqs], int32 + prompt_len_ptr, # [max_num_reqs], int32 +): + batch_idx = tl.program_id(0) + req_state_idx = tl.load(idx_mapping_ptr + batch_idx) + if req_state_idx < 0: + return + + trace_len = tl.load(trace_len_ptr + req_state_idx) + if trace_len <= 0: + return + + # The token being sampled now is output token number + # (total_len - prompt_len): total_len reflects tokens committed through the + # previous step (post_update runs after sampling). + step = tl.load(total_len_ptr + req_state_idx) - tl.load( + prompt_len_ptr + req_state_idx + ) + if step < 0 or step >= trace_len: + return + + token_id = tl.load( + trace_token_ids_ptr + req_state_idx * trace_token_ids_stride + step + ) + tl.store(sampled_ptr + batch_idx, token_id.to(tl.int64)) + + +def apply_trace_tokens( + sampled: torch.Tensor, + idx_mapping: torch.Tensor, + trace_token_ids: torch.Tensor, + trace_len: torch.Tensor, + total_len: torch.Tensor, + prompt_len: torch.Tensor, +) -> None: + """Overwrite ``sampled`` in place with trace tokens for the current step.""" + num_reqs = idx_mapping.shape[0] + _trace_replay_kernel[(num_reqs,)]( + sampled, + idx_mapping, + trace_token_ids, + trace_token_ids.stride(0), + trace_len, + total_len, + prompt_len, + )