[Core][V1] Support trace_decode_token_ids for deterministic decode replay (#46701)
Signed-off-by: ianzxliu <ianzxliu@tencent.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Zhixia Liu <liuzhixia.nju@gmail.com> Co-authored-by: ianzxliu <ianzxliu@tencent.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
ianzxliu
Claude
aoshen02
Codex
Nick Hill
parent
754e1c3de4
commit
a1c5b1fd9f
@@ -262,6 +262,12 @@ class ModelConfig:
|
||||
equivalent exponential-race sampling. FP64 preserves lower-tail sampling
|
||||
events that fp32 uniform/exponential draws can truncate, at the cost of
|
||||
significantly lower throughput on most GPUs."""
|
||||
enable_trace_replay: bool = False
|
||||
"""Whether to allow requests to set
|
||||
`SamplingParams.trace_decode_token_ids`, which forces decoding to follow a
|
||||
predetermined token sequence while still computing real logprobs. Reserved
|
||||
for debugging and RL workflows: enabling it reserves a per-request trace
|
||||
buffer, so it is off by default."""
|
||||
disable_sliding_window: bool = False
|
||||
"""Whether to disable sliding window. If True, we will disable the sliding
|
||||
window functionality of the model, capping to sliding window size. If the
|
||||
@@ -424,6 +430,7 @@ class ModelConfig:
|
||||
"return_sampling_mask",
|
||||
"logprobs_mode",
|
||||
"use_fp64_gumbel",
|
||||
"enable_trace_replay",
|
||||
"disable_cascade_attn",
|
||||
"skip_tokenizer_init",
|
||||
"served_model_name",
|
||||
|
||||
@@ -1091,6 +1091,13 @@ class VllmConfig:
|
||||
"are normalized over the same nucleus as the sampling mask"
|
||||
)
|
||||
|
||||
def _verify_trace_replay_config(self) -> None:
|
||||
model_config = self.model_config
|
||||
if model_config is None or not model_config.enable_trace_replay:
|
||||
return
|
||||
if not self.use_v2_model_runner:
|
||||
raise ValueError("trace replay requires Model Runner V2")
|
||||
|
||||
def __post_init__(self):
|
||||
"""Verify configs are valid & consistent with each other."""
|
||||
|
||||
@@ -1141,6 +1148,7 @@ class VllmConfig:
|
||||
)
|
||||
|
||||
self._verify_sampling_replay_config()
|
||||
self._verify_trace_replay_config()
|
||||
|
||||
if self.lora_config is not None:
|
||||
self.lora_config.verify_with_model_config(self.model_config)
|
||||
|
||||
@@ -543,6 +543,7 @@ class EngineArgs:
|
||||
max_logprobs: int = ModelConfig.max_logprobs
|
||||
logprobs_mode: LogprobsMode = ModelConfig.logprobs_mode
|
||||
use_fp64_gumbel: bool = ModelConfig.use_fp64_gumbel
|
||||
enable_trace_replay: bool = ModelConfig.enable_trace_replay
|
||||
disable_log_stats: bool = False
|
||||
aggregate_engine_logging: bool = False
|
||||
revision: str | None = ModelConfig.revision
|
||||
@@ -884,6 +885,9 @@ class EngineArgs:
|
||||
model_group.add_argument("--max-logprobs", **model_kwargs["max_logprobs"])
|
||||
model_group.add_argument("--logprobs-mode", **model_kwargs["logprobs_mode"])
|
||||
model_group.add_argument("--use-fp64-gumbel", **model_kwargs["use_fp64_gumbel"])
|
||||
model_group.add_argument(
|
||||
"--enable-trace-replay", **model_kwargs["enable_trace_replay"]
|
||||
)
|
||||
model_group.add_argument(
|
||||
"--disable-sliding-window", **model_kwargs["disable_sliding_window"]
|
||||
)
|
||||
@@ -1760,6 +1764,7 @@ class EngineArgs:
|
||||
max_logprobs=self.max_logprobs,
|
||||
logprobs_mode=self.logprobs_mode,
|
||||
use_fp64_gumbel=self.use_fp64_gumbel,
|
||||
enable_trace_replay=self.enable_trace_replay,
|
||||
disable_sliding_window=self.disable_sliding_window,
|
||||
disable_cascade_attn=self.disable_cascade_attn,
|
||||
skip_tokenizer_init=self.skip_tokenizer_init,
|
||||
|
||||
+72
-8
@@ -346,14 +346,6 @@ class SamplingParams(
|
||||
"""Arbitrary additional args, that can be used by custom sampling
|
||||
implementations, plugins, etc. Not used by any in-tree sampling
|
||||
implementations."""
|
||||
routed_experts_prompt_start: int = 0
|
||||
"""When enable_return_routed_experts is active, skip the first
|
||||
routed_experts_prompt_start prompt tokens from the returned routing
|
||||
data. In multi-turn agent scenarios, set this to the length of the
|
||||
already-returned prefix to avoid duplicating routing for prompt tokens
|
||||
covered by earlier turns. Default 0 returns routing for all prompt
|
||||
tokens."""
|
||||
|
||||
# Fields used for bad words
|
||||
bad_words: list[str] | None = None
|
||||
"""Words that are not allowed to be generated. More precisely, only the
|
||||
@@ -373,6 +365,19 @@ class SamplingParams(
|
||||
'\\emoji \\emoji \\emoji ...'). This feature can detect such behavior
|
||||
and terminate early, saving time and tokens."""
|
||||
|
||||
# Debugging / RL-specific parameters. Not intended for production serving.
|
||||
routed_experts_prompt_start: int = 0
|
||||
"""When enable_return_routed_experts is active, skip the first
|
||||
routed_experts_prompt_start prompt tokens from the returned routing
|
||||
data. In multi-turn agent scenarios, set this to the length of the
|
||||
already-returned prefix to avoid duplicating routing for prompt tokens
|
||||
covered by earlier turns. Default 0 returns routing for all prompt
|
||||
tokens."""
|
||||
trace_decode_token_ids: list[int] | None = None
|
||||
"""If provided, forces the engine to emit this predetermined sequence of
|
||||
token IDs during decoding instead of sampling randomly. Real logprobs are
|
||||
still computed. Conflict checking is performed at the engine level."""
|
||||
|
||||
@staticmethod
|
||||
def from_optional(
|
||||
n: int | None = 1,
|
||||
@@ -407,6 +412,8 @@ class SamplingParams(
|
||||
repetition_detection: RepetitionDetectionParams | None = None,
|
||||
logprob_token_ids: list[int] | None = None,
|
||||
routed_experts_prompt_start: int = 0,
|
||||
# Debugging / RL-specific parameters.
|
||||
trace_decode_token_ids: list[int] | None = None,
|
||||
) -> "SamplingParams":
|
||||
if logit_bias is not None:
|
||||
# Fast path uses a dict comprehension; on failure we iterate once
|
||||
@@ -470,6 +477,7 @@ class SamplingParams(
|
||||
skip_clone=skip_clone,
|
||||
repetition_detection=repetition_detection,
|
||||
routed_experts_prompt_start=routed_experts_prompt_start,
|
||||
trace_decode_token_ids=trace_decode_token_ids,
|
||||
)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
@@ -780,6 +788,7 @@ class SamplingParams(
|
||||
) -> None:
|
||||
self._validate_logprobs(model_config)
|
||||
self._validate_logit_bias(model_config)
|
||||
self._validate_trace_replay(model_config, speculative_config)
|
||||
self._validate_logits_processors(model_config)
|
||||
self._validate_allowed_token_ids(tokenizer)
|
||||
self._validate_spec_decode(speculative_config)
|
||||
@@ -870,6 +879,61 @@ class SamplingParams(
|
||||
value=invalid_token_ids,
|
||||
)
|
||||
|
||||
def _validate_trace_replay(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
speculative_config: SpeculativeConfig | None,
|
||||
) -> None:
|
||||
"""Validate trace replay request compatibility."""
|
||||
if self.trace_decode_token_ids is None:
|
||||
return
|
||||
|
||||
if len(self.trace_decode_token_ids) == 0:
|
||||
raise ValueError("trace_decode_token_ids must be a non-empty list.")
|
||||
if self.n != 1:
|
||||
raise ValueError("trace_decode_token_ids requires n=1.")
|
||||
if not all(isinstance(t, int) and t >= 0 for t in self.trace_decode_token_ids):
|
||||
raise ValueError(
|
||||
"trace_decode_token_ids must contain non-negative integers."
|
||||
)
|
||||
|
||||
if self.prompt_logprobs is not None:
|
||||
raise ValueError(
|
||||
"trace_decode_token_ids is not supported with prompt_logprobs."
|
||||
)
|
||||
if speculative_config is not None:
|
||||
raise ValueError(
|
||||
"trace_decode_token_ids is not supported with speculative decoding."
|
||||
)
|
||||
if self.structured_outputs is not None:
|
||||
raise ValueError(
|
||||
"trace_decode_token_ids is not supported with structured outputs."
|
||||
)
|
||||
if self.repetition_detection is not None:
|
||||
raise ValueError(
|
||||
"trace_decode_token_ids is not supported with repetition_detection."
|
||||
)
|
||||
if self.thinking_token_budget is not None:
|
||||
raise ValueError(
|
||||
"trace_decode_token_ids is not supported with thinking_token_budget."
|
||||
)
|
||||
if self.bad_words:
|
||||
raise ValueError("trace_decode_token_ids is not supported with bad_words.")
|
||||
|
||||
vocab_size = model_config.get_vocab_size()
|
||||
invalid_token_ids = [
|
||||
token_id
|
||||
for token_id in self.trace_decode_token_ids
|
||||
if token_id < 0 or token_id >= vocab_size
|
||||
]
|
||||
if invalid_token_ids:
|
||||
raise VLLMValidationError(
|
||||
f"token_id(s) {invalid_token_ids} in trace_decode_token_ids "
|
||||
f"contain out-of-vocab token ids. Vocabulary size: {vocab_size}",
|
||||
parameter="trace_decode_token_ids",
|
||||
value=invalid_token_ids,
|
||||
)
|
||||
|
||||
def _validate_logits_processors(self, model_config: ModelConfig) -> None:
|
||||
from vllm.v1.sample.logits_processor import (
|
||||
validate_logits_processors_parameters,
|
||||
|
||||
@@ -51,7 +51,6 @@ class InputProcessor:
|
||||
self.speculative_config = vllm_config.speculative_config
|
||||
self.structured_outputs_config = vllm_config.structured_outputs_config
|
||||
self.observability_config = vllm_config.observability_config
|
||||
self.use_v2_model_runner = vllm_config.use_v2_model_runner
|
||||
|
||||
self.generation_config_fields = model_config.try_get_generation_config()
|
||||
|
||||
@@ -128,6 +127,15 @@ class InputProcessor:
|
||||
"not configured. Please set --reasoning-parser "
|
||||
"and/or --reasoning-config to use thinking_token_budget."
|
||||
)
|
||||
if (
|
||||
params.trace_decode_token_ids
|
||||
and not self.model_config.enable_trace_replay
|
||||
):
|
||||
raise VLLMValidationError(
|
||||
"trace_decode_token_ids is set but trace replay is not "
|
||||
"enabled. Start the engine with --enable-trace-replay "
|
||||
"to use it."
|
||||
)
|
||||
elif isinstance(params, PoolingParams):
|
||||
supported_pooling_tasks = [
|
||||
task for task in supported_tasks if task in POOLING_TASKS
|
||||
@@ -156,6 +164,30 @@ class InputProcessor:
|
||||
f"but got {type(params).__name__}"
|
||||
)
|
||||
|
||||
def _normalize_trace_replay_params(
|
||||
self, sampling_params: SamplingParams, prompt_len: int
|
||||
) -> None:
|
||||
"""Apply trace replay's generation semantics to request-local params."""
|
||||
trace_token_ids = sampling_params.trace_decode_token_ids
|
||||
assert trace_token_ids
|
||||
assert sampling_params.max_tokens is not None
|
||||
|
||||
max_trace_len = max(self.model_config.max_model_len - prompt_len, 1)
|
||||
trace_token_ids = trace_token_ids[:max_trace_len]
|
||||
sampling_params.trace_decode_token_ids = trace_token_ids
|
||||
|
||||
# Apply this after the generation config so its EOS token cannot stop
|
||||
# replay before the trace is exhausted.
|
||||
sampling_params.max_tokens = min(
|
||||
len(trace_token_ids), sampling_params.max_tokens
|
||||
)
|
||||
sampling_params.min_tokens = 0
|
||||
sampling_params.ignore_eos = True
|
||||
sampling_params._eos_token_id = None
|
||||
sampling_params.stop = []
|
||||
sampling_params.stop_token_ids = []
|
||||
sampling_params._all_stop_token_ids = set()
|
||||
|
||||
def _validate_lora(self, lora_request: LoRARequest | None) -> None:
|
||||
if lora_request is None:
|
||||
return
|
||||
@@ -340,6 +372,13 @@ class InputProcessor:
|
||||
)
|
||||
if self.tokenizer is not None:
|
||||
sampling_params.update_from_tokenizer(self.tokenizer)
|
||||
if sampling_params.trace_decode_token_ids:
|
||||
self._normalize_trace_replay_params(
|
||||
sampling_params,
|
||||
length_from_prompt_token_ids_or_embeds(
|
||||
prompt_token_ids, prompt_embeds
|
||||
),
|
||||
)
|
||||
else:
|
||||
pooling_params = params.clone()
|
||||
|
||||
|
||||
@@ -408,6 +408,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
logprobs_mode=self.model_config.logprobs_mode,
|
||||
num_speculative_tokens=self.decode_query_len,
|
||||
use_fp64_gumbel=self.model_config.use_fp64_gumbel,
|
||||
enable_trace_replay=self.model_config.enable_trace_replay,
|
||||
reasoning_config=self.vllm_config.reasoning_config,
|
||||
return_sampling_mask=self.model_config.return_sampling_mask,
|
||||
)
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.v1.worker.gpu.sample.output import SamplerOutput, SamplingMaskTensors
|
||||
from vllm.v1.worker.gpu.sample.penalties import PenaltiesState
|
||||
from vllm.v1.worker.gpu.sample.states import NO_LOGPROBS, SamplingStates
|
||||
from vllm.v1.worker.gpu.sample.thinking_budget import ThinkingBudgetState
|
||||
from vllm.v1.worker.gpu.sample.trace_replay import TraceReplayState
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
|
||||
@@ -39,6 +40,7 @@ class Sampler:
|
||||
logprobs_mode: LogprobsMode = "raw_logprobs",
|
||||
num_speculative_tokens: int = 1,
|
||||
use_fp64_gumbel: bool = False,
|
||||
enable_trace_replay: bool = False,
|
||||
reasoning_config: ReasoningConfig | None = None,
|
||||
return_sampling_mask: bool = False,
|
||||
):
|
||||
@@ -53,6 +55,9 @@ class Sampler:
|
||||
self.bad_words_state = BadWordsState(req_states)
|
||||
self.logprob_token_ids_state = LogprobTokenIdsState(max_num_reqs, device)
|
||||
self.thinking_budget_state = ThinkingBudgetState(req_states, reasoning_config)
|
||||
self.trace_replay_state = (
|
||||
TraceReplayState(req_states) if enable_trace_replay else None
|
||||
)
|
||||
self.needs_logits_processing = np.zeros(max_num_reqs, dtype=bool)
|
||||
self.num_speculative_tokens = num_speculative_tokens
|
||||
self.return_sampling_mask = return_sampling_mask
|
||||
@@ -69,6 +74,8 @@ class Sampler:
|
||||
self.bad_words_state.add_request(req_idx, sampling_params)
|
||||
self.logprob_token_ids_state.add_request(req_idx, sampling_params)
|
||||
self.thinking_budget_state.add_request(req_idx, sampling_params)
|
||||
if self.trace_replay_state is not None:
|
||||
self.trace_replay_state.add_request(req_idx, sampling_params)
|
||||
|
||||
states = self.sampling_states
|
||||
temperature = states.temperature.np[req_idx]
|
||||
@@ -93,6 +100,8 @@ class Sampler:
|
||||
self.bad_words_state.apply_staged_writes()
|
||||
self.logprob_token_ids_state.apply_staged_writes()
|
||||
self.thinking_budget_state.apply_staged_writes()
|
||||
if self.trace_replay_state is not None:
|
||||
self.trace_replay_state.apply_staged_writes()
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
@@ -128,6 +137,11 @@ class Sampler:
|
||||
return_logprobs=return_logprobs,
|
||||
)
|
||||
|
||||
if self.trace_replay_state is not None:
|
||||
# Overwrite sampled tokens with the replay trace up-front so that
|
||||
# computed logprobs reflect the real distribution of the forced token.
|
||||
self.trace_replay_state.apply_trace(sampled, idx_mapping)
|
||||
|
||||
if return_logprobs:
|
||||
if self.logprobs_mode in PROCESSED_LOGPROBS_MODES:
|
||||
logits = processed_logits
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
|
||||
class TraceReplayState:
|
||||
"""Per-request state for inference trace-replay.
|
||||
|
||||
When a request carries ``SamplingParams.trace_decode_token_ids``, the
|
||||
sampler overwrites the sampled token at each decode step with the
|
||||
predetermined trace token, while real logprobs and ranks are still computed
|
||||
from the unmodified logit distribution. The replay step for a request is
|
||||
derived entirely from GPU state (``total_len - prompt_len``), so no CPU
|
||||
synchronization or async placeholder handling is needed.
|
||||
"""
|
||||
|
||||
def __init__(self, req_states: RequestState):
|
||||
self.max_num_reqs = req_states.max_num_reqs
|
||||
self.device = req_states.device
|
||||
self.req_states = req_states
|
||||
self.trace_token_ids = StagedWriteTensor(
|
||||
(self.max_num_reqs, req_states.max_model_len),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
uva_instead_of_gpu=True,
|
||||
)
|
||||
self.trace_len = UvaBackedTensor(self.max_num_reqs, dtype=torch.int32)
|
||||
|
||||
def add_request(self, req_idx: int, sampling_params: SamplingParams) -> None:
|
||||
trace = sampling_params.trace_decode_token_ids
|
||||
if trace is not None:
|
||||
self.trace_len.np[req_idx] = len(trace)
|
||||
self.trace_token_ids.stage_write(req_idx, 0, trace)
|
||||
else:
|
||||
self.trace_len.np[req_idx] = 0
|
||||
|
||||
def apply_staged_writes(self) -> None:
|
||||
self.trace_len.copy_to_uva()
|
||||
self.trace_token_ids.apply_write()
|
||||
|
||||
def apply_trace(
|
||||
self,
|
||||
sampled: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
) -> None:
|
||||
apply_trace_tokens(
|
||||
sampled,
|
||||
idx_mapping,
|
||||
self.trace_token_ids.gpu,
|
||||
self.trace_len.gpu,
|
||||
self.req_states.total_len.gpu,
|
||||
self.req_states.prompt_len.gpu,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _trace_replay_kernel(
|
||||
sampled_ptr, # [num_reqs], int64, mutated in place
|
||||
idx_mapping_ptr, # [num_reqs] batch_idx -> req_state_idx
|
||||
trace_token_ids_ptr, # [max_num_reqs, max_model_len], int32
|
||||
trace_token_ids_stride,
|
||||
trace_len_ptr, # [max_num_reqs], int32
|
||||
total_len_ptr, # [max_num_reqs], int32
|
||||
prompt_len_ptr, # [max_num_reqs], int32
|
||||
):
|
||||
batch_idx = tl.program_id(0)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + batch_idx)
|
||||
if req_state_idx < 0:
|
||||
return
|
||||
|
||||
trace_len = tl.load(trace_len_ptr + req_state_idx)
|
||||
if trace_len <= 0:
|
||||
return
|
||||
|
||||
# The token being sampled now is output token number
|
||||
# (total_len - prompt_len): total_len reflects tokens committed through the
|
||||
# previous step (post_update runs after sampling).
|
||||
step = tl.load(total_len_ptr + req_state_idx) - tl.load(
|
||||
prompt_len_ptr + req_state_idx
|
||||
)
|
||||
if step < 0 or step >= trace_len:
|
||||
return
|
||||
|
||||
token_id = tl.load(
|
||||
trace_token_ids_ptr + req_state_idx * trace_token_ids_stride + step
|
||||
)
|
||||
tl.store(sampled_ptr + batch_idx, token_id.to(tl.int64))
|
||||
|
||||
|
||||
def apply_trace_tokens(
|
||||
sampled: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
trace_token_ids: torch.Tensor,
|
||||
trace_len: torch.Tensor,
|
||||
total_len: torch.Tensor,
|
||||
prompt_len: torch.Tensor,
|
||||
) -> None:
|
||||
"""Overwrite ``sampled`` in place with trace tokens for the current step."""
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
_trace_replay_kernel[(num_reqs,)](
|
||||
sampled,
|
||||
idx_mapping,
|
||||
trace_token_ids,
|
||||
trace_token_ids.stride(0),
|
||||
trace_len,
|
||||
total_len,
|
||||
prompt_len,
|
||||
)
|
||||
Reference in New Issue
Block a user