[Core][V1] Support trace_decode_token_ids for deterministic decode replay (#46701)
Signed-off-by: ianzxliu <ianzxliu@tencent.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Zhixia Liu <liuzhixia.nju@gmail.com> Co-authored-by: ianzxliu <ianzxliu@tencent.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
ianzxliu
Claude
aoshen02
Codex
Nick Hill
parent
754e1c3de4
commit
a1c5b1fd9f
@@ -26,6 +26,7 @@ from vllm.v1.worker.gpu.sample.output import SamplerOutput, SamplingMaskTensors
|
||||
from vllm.v1.worker.gpu.sample.penalties import PenaltiesState
|
||||
from vllm.v1.worker.gpu.sample.states import NO_LOGPROBS, SamplingStates
|
||||
from vllm.v1.worker.gpu.sample.thinking_budget import ThinkingBudgetState
|
||||
from vllm.v1.worker.gpu.sample.trace_replay import TraceReplayState
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
|
||||
@@ -39,6 +40,7 @@ class Sampler:
|
||||
logprobs_mode: LogprobsMode = "raw_logprobs",
|
||||
num_speculative_tokens: int = 1,
|
||||
use_fp64_gumbel: bool = False,
|
||||
enable_trace_replay: bool = False,
|
||||
reasoning_config: ReasoningConfig | None = None,
|
||||
return_sampling_mask: bool = False,
|
||||
):
|
||||
@@ -53,6 +55,9 @@ class Sampler:
|
||||
self.bad_words_state = BadWordsState(req_states)
|
||||
self.logprob_token_ids_state = LogprobTokenIdsState(max_num_reqs, device)
|
||||
self.thinking_budget_state = ThinkingBudgetState(req_states, reasoning_config)
|
||||
self.trace_replay_state = (
|
||||
TraceReplayState(req_states) if enable_trace_replay else None
|
||||
)
|
||||
self.needs_logits_processing = np.zeros(max_num_reqs, dtype=bool)
|
||||
self.num_speculative_tokens = num_speculative_tokens
|
||||
self.return_sampling_mask = return_sampling_mask
|
||||
@@ -69,6 +74,8 @@ class Sampler:
|
||||
self.bad_words_state.add_request(req_idx, sampling_params)
|
||||
self.logprob_token_ids_state.add_request(req_idx, sampling_params)
|
||||
self.thinking_budget_state.add_request(req_idx, sampling_params)
|
||||
if self.trace_replay_state is not None:
|
||||
self.trace_replay_state.add_request(req_idx, sampling_params)
|
||||
|
||||
states = self.sampling_states
|
||||
temperature = states.temperature.np[req_idx]
|
||||
@@ -93,6 +100,8 @@ class Sampler:
|
||||
self.bad_words_state.apply_staged_writes()
|
||||
self.logprob_token_ids_state.apply_staged_writes()
|
||||
self.thinking_budget_state.apply_staged_writes()
|
||||
if self.trace_replay_state is not None:
|
||||
self.trace_replay_state.apply_staged_writes()
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
@@ -128,6 +137,11 @@ class Sampler:
|
||||
return_logprobs=return_logprobs,
|
||||
)
|
||||
|
||||
if self.trace_replay_state is not None:
|
||||
# Overwrite sampled tokens with the replay trace up-front so that
|
||||
# computed logprobs reflect the real distribution of the forced token.
|
||||
self.trace_replay_state.apply_trace(sampled, idx_mapping)
|
||||
|
||||
if return_logprobs:
|
||||
if self.logprobs_mode in PROCESSED_LOGPROBS_MODES:
|
||||
logits = processed_logits
|
||||
|
||||
Reference in New Issue
Block a user