[Core][V1] Support trace_decode_token_ids for deterministic decode replay (#46701)

Signed-off-by: ianzxliu <ianzxliu@tencent.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Zhixia Liu <liuzhixia.nju@gmail.com>
Co-authored-by: ianzxliu <ianzxliu@tencent.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Zhixia Liu
2026-08-20 15:12:43 +08:00
committed by GitHub
co-authored by ianzxliu Claude aoshen02 Codex Nick Hill
parent 754e1c3de4
commit a1c5b1fd9f
13 changed files with 931 additions and 9 deletions
+7
View File
@@ -262,6 +262,12 @@ class ModelConfig:
equivalent exponential-race sampling. FP64 preserves lower-tail sampling
events that fp32 uniform/exponential draws can truncate, at the cost of
significantly lower throughput on most GPUs."""
enable_trace_replay: bool = False
"""Whether to allow requests to set
`SamplingParams.trace_decode_token_ids`, which forces decoding to follow a
predetermined token sequence while still computing real logprobs. Reserved
for debugging and RL workflows: enabling it reserves a per-request trace
buffer, so it is off by default."""
disable_sliding_window: bool = False
"""Whether to disable sliding window. If True, we will disable the sliding
window functionality of the model, capping to sliding window size. If the
@@ -424,6 +430,7 @@ class ModelConfig:
"return_sampling_mask",
"logprobs_mode",
"use_fp64_gumbel",
"enable_trace_replay",
"disable_cascade_attn",
"skip_tokenizer_init",
"served_model_name",
+8
View File
@@ -1091,6 +1091,13 @@ class VllmConfig:
"are normalized over the same nucleus as the sampling mask"
)
def _verify_trace_replay_config(self) -> None:
model_config = self.model_config
if model_config is None or not model_config.enable_trace_replay:
return
if not self.use_v2_model_runner:
raise ValueError("trace replay requires Model Runner V2")
def __post_init__(self):
"""Verify configs are valid & consistent with each other."""
@@ -1141,6 +1148,7 @@ class VllmConfig:
)
self._verify_sampling_replay_config()
self._verify_trace_replay_config()
if self.lora_config is not None:
self.lora_config.verify_with_model_config(self.model_config)
+5
View File
@@ -543,6 +543,7 @@ class EngineArgs:
max_logprobs: int = ModelConfig.max_logprobs
logprobs_mode: LogprobsMode = ModelConfig.logprobs_mode
use_fp64_gumbel: bool = ModelConfig.use_fp64_gumbel
enable_trace_replay: bool = ModelConfig.enable_trace_replay
disable_log_stats: bool = False
aggregate_engine_logging: bool = False
revision: str | None = ModelConfig.revision
@@ -884,6 +885,9 @@ class EngineArgs:
model_group.add_argument("--max-logprobs", **model_kwargs["max_logprobs"])
model_group.add_argument("--logprobs-mode", **model_kwargs["logprobs_mode"])
model_group.add_argument("--use-fp64-gumbel", **model_kwargs["use_fp64_gumbel"])
model_group.add_argument(
"--enable-trace-replay", **model_kwargs["enable_trace_replay"]
)
model_group.add_argument(
"--disable-sliding-window", **model_kwargs["disable_sliding_window"]
)
@@ -1760,6 +1764,7 @@ class EngineArgs:
max_logprobs=self.max_logprobs,
logprobs_mode=self.logprobs_mode,
use_fp64_gumbel=self.use_fp64_gumbel,
enable_trace_replay=self.enable_trace_replay,
disable_sliding_window=self.disable_sliding_window,
disable_cascade_attn=self.disable_cascade_attn,
skip_tokenizer_init=self.skip_tokenizer_init,
+72 -8
View File
@@ -346,14 +346,6 @@ class SamplingParams(
"""Arbitrary additional args, that can be used by custom sampling
implementations, plugins, etc. Not used by any in-tree sampling
implementations."""
routed_experts_prompt_start: int = 0
"""When enable_return_routed_experts is active, skip the first
routed_experts_prompt_start prompt tokens from the returned routing
data. In multi-turn agent scenarios, set this to the length of the
already-returned prefix to avoid duplicating routing for prompt tokens
covered by earlier turns. Default 0 returns routing for all prompt
tokens."""
# Fields used for bad words
bad_words: list[str] | None = None
"""Words that are not allowed to be generated. More precisely, only the
@@ -373,6 +365,19 @@ class SamplingParams(
'\\emoji \\emoji \\emoji ...'). This feature can detect such behavior
and terminate early, saving time and tokens."""
# Debugging / RL-specific parameters. Not intended for production serving.
routed_experts_prompt_start: int = 0
"""When enable_return_routed_experts is active, skip the first
routed_experts_prompt_start prompt tokens from the returned routing
data. In multi-turn agent scenarios, set this to the length of the
already-returned prefix to avoid duplicating routing for prompt tokens
covered by earlier turns. Default 0 returns routing for all prompt
tokens."""
trace_decode_token_ids: list[int] | None = None
"""If provided, forces the engine to emit this predetermined sequence of
token IDs during decoding instead of sampling randomly. Real logprobs are
still computed. Conflict checking is performed at the engine level."""
@staticmethod
def from_optional(
n: int | None = 1,
@@ -407,6 +412,8 @@ class SamplingParams(
repetition_detection: RepetitionDetectionParams | None = None,
logprob_token_ids: list[int] | None = None,
routed_experts_prompt_start: int = 0,
# Debugging / RL-specific parameters.
trace_decode_token_ids: list[int] | None = None,
) -> "SamplingParams":
if logit_bias is not None:
# Fast path uses a dict comprehension; on failure we iterate once
@@ -470,6 +477,7 @@ class SamplingParams(
skip_clone=skip_clone,
repetition_detection=repetition_detection,
routed_experts_prompt_start=routed_experts_prompt_start,
trace_decode_token_ids=trace_decode_token_ids,
)
def __post_init__(self) -> None:
@@ -780,6 +788,7 @@ class SamplingParams(
) -> None:
self._validate_logprobs(model_config)
self._validate_logit_bias(model_config)
self._validate_trace_replay(model_config, speculative_config)
self._validate_logits_processors(model_config)
self._validate_allowed_token_ids(tokenizer)
self._validate_spec_decode(speculative_config)
@@ -870,6 +879,61 @@ class SamplingParams(
value=invalid_token_ids,
)
def _validate_trace_replay(
self,
model_config: ModelConfig,
speculative_config: SpeculativeConfig | None,
) -> None:
"""Validate trace replay request compatibility."""
if self.trace_decode_token_ids is None:
return
if len(self.trace_decode_token_ids) == 0:
raise ValueError("trace_decode_token_ids must be a non-empty list.")
if self.n != 1:
raise ValueError("trace_decode_token_ids requires n=1.")
if not all(isinstance(t, int) and t >= 0 for t in self.trace_decode_token_ids):
raise ValueError(
"trace_decode_token_ids must contain non-negative integers."
)
if self.prompt_logprobs is not None:
raise ValueError(
"trace_decode_token_ids is not supported with prompt_logprobs."
)
if speculative_config is not None:
raise ValueError(
"trace_decode_token_ids is not supported with speculative decoding."
)
if self.structured_outputs is not None:
raise ValueError(
"trace_decode_token_ids is not supported with structured outputs."
)
if self.repetition_detection is not None:
raise ValueError(
"trace_decode_token_ids is not supported with repetition_detection."
)
if self.thinking_token_budget is not None:
raise ValueError(
"trace_decode_token_ids is not supported with thinking_token_budget."
)
if self.bad_words:
raise ValueError("trace_decode_token_ids is not supported with bad_words.")
vocab_size = model_config.get_vocab_size()
invalid_token_ids = [
token_id
for token_id in self.trace_decode_token_ids
if token_id < 0 or token_id >= vocab_size
]
if invalid_token_ids:
raise VLLMValidationError(
f"token_id(s) {invalid_token_ids} in trace_decode_token_ids "
f"contain out-of-vocab token ids. Vocabulary size: {vocab_size}",
parameter="trace_decode_token_ids",
value=invalid_token_ids,
)
def _validate_logits_processors(self, model_config: ModelConfig) -> None:
from vllm.v1.sample.logits_processor import (
validate_logits_processors_parameters,
+40 -1
View File
@@ -51,7 +51,6 @@ class InputProcessor:
self.speculative_config = vllm_config.speculative_config
self.structured_outputs_config = vllm_config.structured_outputs_config
self.observability_config = vllm_config.observability_config
self.use_v2_model_runner = vllm_config.use_v2_model_runner
self.generation_config_fields = model_config.try_get_generation_config()
@@ -128,6 +127,15 @@ class InputProcessor:
"not configured. Please set --reasoning-parser "
"and/or --reasoning-config to use thinking_token_budget."
)
if (
params.trace_decode_token_ids
and not self.model_config.enable_trace_replay
):
raise VLLMValidationError(
"trace_decode_token_ids is set but trace replay is not "
"enabled. Start the engine with --enable-trace-replay "
"to use it."
)
elif isinstance(params, PoolingParams):
supported_pooling_tasks = [
task for task in supported_tasks if task in POOLING_TASKS
@@ -156,6 +164,30 @@ class InputProcessor:
f"but got {type(params).__name__}"
)
def _normalize_trace_replay_params(
self, sampling_params: SamplingParams, prompt_len: int
) -> None:
"""Apply trace replay's generation semantics to request-local params."""
trace_token_ids = sampling_params.trace_decode_token_ids
assert trace_token_ids
assert sampling_params.max_tokens is not None
max_trace_len = max(self.model_config.max_model_len - prompt_len, 1)
trace_token_ids = trace_token_ids[:max_trace_len]
sampling_params.trace_decode_token_ids = trace_token_ids
# Apply this after the generation config so its EOS token cannot stop
# replay before the trace is exhausted.
sampling_params.max_tokens = min(
len(trace_token_ids), sampling_params.max_tokens
)
sampling_params.min_tokens = 0
sampling_params.ignore_eos = True
sampling_params._eos_token_id = None
sampling_params.stop = []
sampling_params.stop_token_ids = []
sampling_params._all_stop_token_ids = set()
def _validate_lora(self, lora_request: LoRARequest | None) -> None:
if lora_request is None:
return
@@ -340,6 +372,13 @@ class InputProcessor:
)
if self.tokenizer is not None:
sampling_params.update_from_tokenizer(self.tokenizer)
if sampling_params.trace_decode_token_ids:
self._normalize_trace_replay_params(
sampling_params,
length_from_prompt_token_ids_or_embeds(
prompt_token_ids, prompt_embeds
),
)
else:
pooling_params = params.clone()
+1
View File
@@ -408,6 +408,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
logprobs_mode=self.model_config.logprobs_mode,
num_speculative_tokens=self.decode_query_len,
use_fp64_gumbel=self.model_config.use_fp64_gumbel,
enable_trace_replay=self.model_config.enable_trace_replay,
reasoning_config=self.vllm_config.reasoning_config,
return_sampling_mask=self.model_config.return_sampling_mask,
)
+14
View File
@@ -26,6 +26,7 @@ from vllm.v1.worker.gpu.sample.output import SamplerOutput, SamplingMaskTensors
from vllm.v1.worker.gpu.sample.penalties import PenaltiesState
from vllm.v1.worker.gpu.sample.states import NO_LOGPROBS, SamplingStates
from vllm.v1.worker.gpu.sample.thinking_budget import ThinkingBudgetState
from vllm.v1.worker.gpu.sample.trace_replay import TraceReplayState
from vllm.v1.worker.gpu.states import RequestState
@@ -39,6 +40,7 @@ class Sampler:
logprobs_mode: LogprobsMode = "raw_logprobs",
num_speculative_tokens: int = 1,
use_fp64_gumbel: bool = False,
enable_trace_replay: bool = False,
reasoning_config: ReasoningConfig | None = None,
return_sampling_mask: bool = False,
):
@@ -53,6 +55,9 @@ class Sampler:
self.bad_words_state = BadWordsState(req_states)
self.logprob_token_ids_state = LogprobTokenIdsState(max_num_reqs, device)
self.thinking_budget_state = ThinkingBudgetState(req_states, reasoning_config)
self.trace_replay_state = (
TraceReplayState(req_states) if enable_trace_replay else None
)
self.needs_logits_processing = np.zeros(max_num_reqs, dtype=bool)
self.num_speculative_tokens = num_speculative_tokens
self.return_sampling_mask = return_sampling_mask
@@ -69,6 +74,8 @@ class Sampler:
self.bad_words_state.add_request(req_idx, sampling_params)
self.logprob_token_ids_state.add_request(req_idx, sampling_params)
self.thinking_budget_state.add_request(req_idx, sampling_params)
if self.trace_replay_state is not None:
self.trace_replay_state.add_request(req_idx, sampling_params)
states = self.sampling_states
temperature = states.temperature.np[req_idx]
@@ -93,6 +100,8 @@ class Sampler:
self.bad_words_state.apply_staged_writes()
self.logprob_token_ids_state.apply_staged_writes()
self.thinking_budget_state.apply_staged_writes()
if self.trace_replay_state is not None:
self.trace_replay_state.apply_staged_writes()
def __call__(
self,
@@ -128,6 +137,11 @@ class Sampler:
return_logprobs=return_logprobs,
)
if self.trace_replay_state is not None:
# Overwrite sampled tokens with the replay trace up-front so that
# computed logprobs reflect the real distribution of the forced token.
self.trace_replay_state.apply_trace(sampled, idx_mapping)
if return_logprobs:
if self.logprobs_mode in PROCESSED_LOGPROBS_MODES:
logits = processed_logits
+113
View File
@@ -0,0 +1,113 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.sampling_params import SamplingParams
from vllm.triton_utils import tl, triton
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor
from vllm.v1.worker.gpu.states import RequestState
class TraceReplayState:
"""Per-request state for inference trace-replay.
When a request carries ``SamplingParams.trace_decode_token_ids``, the
sampler overwrites the sampled token at each decode step with the
predetermined trace token, while real logprobs and ranks are still computed
from the unmodified logit distribution. The replay step for a request is
derived entirely from GPU state (``total_len - prompt_len``), so no CPU
synchronization or async placeholder handling is needed.
"""
def __init__(self, req_states: RequestState):
self.max_num_reqs = req_states.max_num_reqs
self.device = req_states.device
self.req_states = req_states
self.trace_token_ids = StagedWriteTensor(
(self.max_num_reqs, req_states.max_model_len),
dtype=torch.int32,
device=self.device,
uva_instead_of_gpu=True,
)
self.trace_len = UvaBackedTensor(self.max_num_reqs, dtype=torch.int32)
def add_request(self, req_idx: int, sampling_params: SamplingParams) -> None:
trace = sampling_params.trace_decode_token_ids
if trace is not None:
self.trace_len.np[req_idx] = len(trace)
self.trace_token_ids.stage_write(req_idx, 0, trace)
else:
self.trace_len.np[req_idx] = 0
def apply_staged_writes(self) -> None:
self.trace_len.copy_to_uva()
self.trace_token_ids.apply_write()
def apply_trace(
self,
sampled: torch.Tensor,
idx_mapping: torch.Tensor,
) -> None:
apply_trace_tokens(
sampled,
idx_mapping,
self.trace_token_ids.gpu,
self.trace_len.gpu,
self.req_states.total_len.gpu,
self.req_states.prompt_len.gpu,
)
@triton.jit
def _trace_replay_kernel(
sampled_ptr, # [num_reqs], int64, mutated in place
idx_mapping_ptr, # [num_reqs] batch_idx -> req_state_idx
trace_token_ids_ptr, # [max_num_reqs, max_model_len], int32
trace_token_ids_stride,
trace_len_ptr, # [max_num_reqs], int32
total_len_ptr, # [max_num_reqs], int32
prompt_len_ptr, # [max_num_reqs], int32
):
batch_idx = tl.program_id(0)
req_state_idx = tl.load(idx_mapping_ptr + batch_idx)
if req_state_idx < 0:
return
trace_len = tl.load(trace_len_ptr + req_state_idx)
if trace_len <= 0:
return
# The token being sampled now is output token number
# (total_len - prompt_len): total_len reflects tokens committed through the
# previous step (post_update runs after sampling).
step = tl.load(total_len_ptr + req_state_idx) - tl.load(
prompt_len_ptr + req_state_idx
)
if step < 0 or step >= trace_len:
return
token_id = tl.load(
trace_token_ids_ptr + req_state_idx * trace_token_ids_stride + step
)
tl.store(sampled_ptr + batch_idx, token_id.to(tl.int64))
def apply_trace_tokens(
sampled: torch.Tensor,
idx_mapping: torch.Tensor,
trace_token_ids: torch.Tensor,
trace_len: torch.Tensor,
total_len: torch.Tensor,
prompt_len: torch.Tensor,
) -> None:
"""Overwrite ``sampled`` in place with trace tokens for the current step."""
num_reqs = idx_mapping.shape[0]
_trace_replay_kernel[(num_reqs,)](
sampled,
idx_mapping,
trace_token_ids,
trace_token_ids.stride(0),
trace_len,
total_len,
prompt_len,
)