[Core][V1] Support trace_decode_token_ids for deterministic decode replay (#46701)
Signed-off-by: ianzxliu <ianzxliu@tencent.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Zhixia Liu <liuzhixia.nju@gmail.com> Co-authored-by: ianzxliu <ianzxliu@tencent.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
ianzxliu
Claude
aoshen02
Codex
Nick Hill
parent
754e1c3de4
commit
a1c5b1fd9f
@@ -26,6 +26,7 @@ from vllm.v1.worker.gpu.sample.output import SamplerOutput, SamplingMaskTensors
|
||||
from vllm.v1.worker.gpu.sample.penalties import PenaltiesState
|
||||
from vllm.v1.worker.gpu.sample.states import NO_LOGPROBS, SamplingStates
|
||||
from vllm.v1.worker.gpu.sample.thinking_budget import ThinkingBudgetState
|
||||
from vllm.v1.worker.gpu.sample.trace_replay import TraceReplayState
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
|
||||
@@ -39,6 +40,7 @@ class Sampler:
|
||||
logprobs_mode: LogprobsMode = "raw_logprobs",
|
||||
num_speculative_tokens: int = 1,
|
||||
use_fp64_gumbel: bool = False,
|
||||
enable_trace_replay: bool = False,
|
||||
reasoning_config: ReasoningConfig | None = None,
|
||||
return_sampling_mask: bool = False,
|
||||
):
|
||||
@@ -53,6 +55,9 @@ class Sampler:
|
||||
self.bad_words_state = BadWordsState(req_states)
|
||||
self.logprob_token_ids_state = LogprobTokenIdsState(max_num_reqs, device)
|
||||
self.thinking_budget_state = ThinkingBudgetState(req_states, reasoning_config)
|
||||
self.trace_replay_state = (
|
||||
TraceReplayState(req_states) if enable_trace_replay else None
|
||||
)
|
||||
self.needs_logits_processing = np.zeros(max_num_reqs, dtype=bool)
|
||||
self.num_speculative_tokens = num_speculative_tokens
|
||||
self.return_sampling_mask = return_sampling_mask
|
||||
@@ -69,6 +74,8 @@ class Sampler:
|
||||
self.bad_words_state.add_request(req_idx, sampling_params)
|
||||
self.logprob_token_ids_state.add_request(req_idx, sampling_params)
|
||||
self.thinking_budget_state.add_request(req_idx, sampling_params)
|
||||
if self.trace_replay_state is not None:
|
||||
self.trace_replay_state.add_request(req_idx, sampling_params)
|
||||
|
||||
states = self.sampling_states
|
||||
temperature = states.temperature.np[req_idx]
|
||||
@@ -93,6 +100,8 @@ class Sampler:
|
||||
self.bad_words_state.apply_staged_writes()
|
||||
self.logprob_token_ids_state.apply_staged_writes()
|
||||
self.thinking_budget_state.apply_staged_writes()
|
||||
if self.trace_replay_state is not None:
|
||||
self.trace_replay_state.apply_staged_writes()
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
@@ -128,6 +137,11 @@ class Sampler:
|
||||
return_logprobs=return_logprobs,
|
||||
)
|
||||
|
||||
if self.trace_replay_state is not None:
|
||||
# Overwrite sampled tokens with the replay trace up-front so that
|
||||
# computed logprobs reflect the real distribution of the forced token.
|
||||
self.trace_replay_state.apply_trace(sampled, idx_mapping)
|
||||
|
||||
if return_logprobs:
|
||||
if self.logprobs_mode in PROCESSED_LOGPROBS_MODES:
|
||||
logits = processed_logits
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
|
||||
class TraceReplayState:
|
||||
"""Per-request state for inference trace-replay.
|
||||
|
||||
When a request carries ``SamplingParams.trace_decode_token_ids``, the
|
||||
sampler overwrites the sampled token at each decode step with the
|
||||
predetermined trace token, while real logprobs and ranks are still computed
|
||||
from the unmodified logit distribution. The replay step for a request is
|
||||
derived entirely from GPU state (``total_len - prompt_len``), so no CPU
|
||||
synchronization or async placeholder handling is needed.
|
||||
"""
|
||||
|
||||
def __init__(self, req_states: RequestState):
|
||||
self.max_num_reqs = req_states.max_num_reqs
|
||||
self.device = req_states.device
|
||||
self.req_states = req_states
|
||||
self.trace_token_ids = StagedWriteTensor(
|
||||
(self.max_num_reqs, req_states.max_model_len),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
uva_instead_of_gpu=True,
|
||||
)
|
||||
self.trace_len = UvaBackedTensor(self.max_num_reqs, dtype=torch.int32)
|
||||
|
||||
def add_request(self, req_idx: int, sampling_params: SamplingParams) -> None:
|
||||
trace = sampling_params.trace_decode_token_ids
|
||||
if trace is not None:
|
||||
self.trace_len.np[req_idx] = len(trace)
|
||||
self.trace_token_ids.stage_write(req_idx, 0, trace)
|
||||
else:
|
||||
self.trace_len.np[req_idx] = 0
|
||||
|
||||
def apply_staged_writes(self) -> None:
|
||||
self.trace_len.copy_to_uva()
|
||||
self.trace_token_ids.apply_write()
|
||||
|
||||
def apply_trace(
|
||||
self,
|
||||
sampled: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
) -> None:
|
||||
apply_trace_tokens(
|
||||
sampled,
|
||||
idx_mapping,
|
||||
self.trace_token_ids.gpu,
|
||||
self.trace_len.gpu,
|
||||
self.req_states.total_len.gpu,
|
||||
self.req_states.prompt_len.gpu,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _trace_replay_kernel(
|
||||
sampled_ptr, # [num_reqs], int64, mutated in place
|
||||
idx_mapping_ptr, # [num_reqs] batch_idx -> req_state_idx
|
||||
trace_token_ids_ptr, # [max_num_reqs, max_model_len], int32
|
||||
trace_token_ids_stride,
|
||||
trace_len_ptr, # [max_num_reqs], int32
|
||||
total_len_ptr, # [max_num_reqs], int32
|
||||
prompt_len_ptr, # [max_num_reqs], int32
|
||||
):
|
||||
batch_idx = tl.program_id(0)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + batch_idx)
|
||||
if req_state_idx < 0:
|
||||
return
|
||||
|
||||
trace_len = tl.load(trace_len_ptr + req_state_idx)
|
||||
if trace_len <= 0:
|
||||
return
|
||||
|
||||
# The token being sampled now is output token number
|
||||
# (total_len - prompt_len): total_len reflects tokens committed through the
|
||||
# previous step (post_update runs after sampling).
|
||||
step = tl.load(total_len_ptr + req_state_idx) - tl.load(
|
||||
prompt_len_ptr + req_state_idx
|
||||
)
|
||||
if step < 0 or step >= trace_len:
|
||||
return
|
||||
|
||||
token_id = tl.load(
|
||||
trace_token_ids_ptr + req_state_idx * trace_token_ids_stride + step
|
||||
)
|
||||
tl.store(sampled_ptr + batch_idx, token_id.to(tl.int64))
|
||||
|
||||
|
||||
def apply_trace_tokens(
|
||||
sampled: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
trace_token_ids: torch.Tensor,
|
||||
trace_len: torch.Tensor,
|
||||
total_len: torch.Tensor,
|
||||
prompt_len: torch.Tensor,
|
||||
) -> None:
|
||||
"""Overwrite ``sampled`` in place with trace tokens for the current step."""
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
_trace_replay_kernel[(num_reqs,)](
|
||||
sampled,
|
||||
idx_mapping,
|
||||
trace_token_ids,
|
||||
trace_token_ids.stride(0),
|
||||
trace_len,
|
||||
total_len,
|
||||
prompt_len,
|
||||
)
|
||||
Reference in New Issue
Block a user