[Core][V1] Support trace_decode_token_ids for deterministic decode replay (#46701)
Signed-off-by: ianzxliu <ianzxliu@tencent.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Zhixia Liu <liuzhixia.nju@gmail.com> Co-authored-by: ianzxliu <ianzxliu@tencent.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
ianzxliu
Claude
aoshen02
Codex
Nick Hill
parent
754e1c3de4
commit
a1c5b1fd9f
@@ -0,0 +1,110 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.v1.core.sched.utils import check_stop
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
def _make_request(params: SamplingParams) -> Request:
|
||||
return Request(
|
||||
request_id="req-0",
|
||||
client_index=0,
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
sampling_params=params,
|
||||
pooling_params=None,
|
||||
arrival_time=time.time(),
|
||||
)
|
||||
|
||||
|
||||
def _normalize(params: SamplingParams, prompt_len: int, max_model_len: int) -> None:
|
||||
processor = SimpleNamespace(
|
||||
model_config=SimpleNamespace(max_model_len=max_model_len)
|
||||
)
|
||||
InputProcessor._normalize_trace_replay_params(processor, params, prompt_len)
|
||||
|
||||
|
||||
def test_normalize_trace_replay_params():
|
||||
params = SamplingParams(
|
||||
max_tokens=16,
|
||||
min_tokens=10,
|
||||
stop_token_ids=[20],
|
||||
trace_decode_token_ids=[10, 20, 30],
|
||||
)
|
||||
params.update_from_generation_config({}, eos_token_id=20)
|
||||
|
||||
_normalize(params, prompt_len=3, max_model_len=128)
|
||||
|
||||
assert params.max_tokens == 3
|
||||
assert params.min_tokens == 0
|
||||
assert params.ignore_eos is True
|
||||
assert params.eos_token_id is None
|
||||
assert params.stop_token_ids == []
|
||||
assert params.all_stop_token_ids == set()
|
||||
|
||||
request = _make_request(params)
|
||||
assert request.max_tokens == 3
|
||||
request.append_output_token_ids([10, 20, 30])
|
||||
assert check_stop(request, max_model_len=128)
|
||||
assert request.status == RequestStatus.FINISHED_LENGTH_CAPPED
|
||||
|
||||
|
||||
def test_trace_longer_than_remaining_context_is_truncated():
|
||||
"""The trace is staged into a max_model_len-wide row, so it must be cut.
|
||||
|
||||
An explicitly set max_tokens is never clamped to max_model_len, so it cannot
|
||||
bound the staged write.
|
||||
"""
|
||||
params = SamplingParams(max_tokens=100, trace_decode_token_ids=list(range(20)))
|
||||
|
||||
_normalize(params, prompt_len=6, max_model_len=10)
|
||||
|
||||
assert params.trace_decode_token_ids == [0, 1, 2, 3]
|
||||
assert params.max_tokens == 4
|
||||
|
||||
|
||||
def _validate(enable_trace_replay: bool) -> None:
|
||||
"""Run _validate_params' trace gating with the rest of verify() stubbed."""
|
||||
processor = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
return_sampling_mask=False,
|
||||
enable_trace_replay=enable_trace_replay,
|
||||
),
|
||||
vllm_config=SimpleNamespace(reasoning_config=None),
|
||||
speculative_config=None,
|
||||
structured_outputs_config=None,
|
||||
tokenizer=None,
|
||||
)
|
||||
params = SamplingParams(trace_decode_token_ids=[1, 2, 3])
|
||||
with patch.object(SamplingParams, "verify"):
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
|
||||
|
||||
def test_trace_request_rejected_when_feature_disabled():
|
||||
with pytest.raises(VLLMValidationError, match="--enable-trace-replay"):
|
||||
_validate(enable_trace_replay=False)
|
||||
|
||||
|
||||
def test_trace_request_accepted_when_feature_enabled():
|
||||
_validate(enable_trace_replay=True)
|
||||
|
||||
|
||||
def test_trace_replay_requires_v2_model_runner():
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(enable_trace_replay=True),
|
||||
use_v2_model_runner=False,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="trace replay requires Model Runner V2"):
|
||||
VllmConfig._verify_trace_replay_config(config)
|
||||
@@ -0,0 +1,111 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SamplingParams field
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_sampling_params_trace_field_defaults_to_none():
|
||||
params = SamplingParams(max_tokens=10)
|
||||
assert params.trace_decode_token_ids is None
|
||||
|
||||
|
||||
def test_sampling_params_trace_field_accepts_list():
|
||||
ids = [100, 200, 300]
|
||||
params = SamplingParams(trace_decode_token_ids=ids)
|
||||
assert params.trace_decode_token_ids == ids
|
||||
|
||||
|
||||
def test_sampling_params_trace_field_preserved_by_clone():
|
||||
ids = [1, 2, 3]
|
||||
params = SamplingParams(trace_decode_token_ids=ids)
|
||||
cloned = params.clone()
|
||||
assert cloned.trace_decode_token_ids == ids
|
||||
assert cloned.trace_decode_token_ids is not params.trace_decode_token_ids
|
||||
|
||||
|
||||
def test_sampling_params_trace_field_rejects_empty_list():
|
||||
params = SamplingParams(trace_decode_token_ids=[])
|
||||
with pytest.raises(ValueError, match="non-empty"):
|
||||
params._validate_trace_replay(
|
||||
_make_model_config(vocab_size=100), speculative_config=None
|
||||
)
|
||||
|
||||
|
||||
def test_sampling_params_trace_field_requires_single_output():
|
||||
params = SamplingParams(n=2, trace_decode_token_ids=[1])
|
||||
with pytest.raises(ValueError, match="requires n=1"):
|
||||
params._validate_trace_replay(
|
||||
_make_model_config(vocab_size=100), speculative_config=None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("invalid_ids", [[-1, 5], [1, "2"]])
|
||||
def test_sampling_params_trace_field_rejects_invalid_token_ids(invalid_ids):
|
||||
params = SamplingParams(trace_decode_token_ids=invalid_ids)
|
||||
with pytest.raises(ValueError, match="non-negative integers"):
|
||||
params._validate_trace_replay(
|
||||
_make_model_config(vocab_size=100), speculative_config=None
|
||||
)
|
||||
|
||||
|
||||
def _make_model_config(vocab_size: int):
|
||||
from unittest.mock import Mock
|
||||
|
||||
model_config = Mock()
|
||||
model_config.get_vocab_size = lambda: vocab_size
|
||||
return model_config
|
||||
|
||||
|
||||
def test_validate_trace_replay_accepts_in_vocab():
|
||||
params = SamplingParams(trace_decode_token_ids=[0, 50, 99])
|
||||
# Should not raise.
|
||||
params._validate_trace_replay(
|
||||
_make_model_config(vocab_size=100), speculative_config=None
|
||||
)
|
||||
|
||||
|
||||
def test_validate_trace_replay_rejects_out_of_vocab():
|
||||
# The non-negative check passes at construction, but the token id exceeds
|
||||
# the vocabulary; verify() must reject it before it reaches the sampler.
|
||||
params = SamplingParams(trace_decode_token_ids=[0, 100])
|
||||
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
|
||||
params._validate_trace_replay(
|
||||
_make_model_config(vocab_size=100), speculative_config=None
|
||||
)
|
||||
|
||||
|
||||
def test_validate_trace_replay_noop_when_unset():
|
||||
params = SamplingParams(max_tokens=4)
|
||||
# Should not raise when the field is unset.
|
||||
params._validate_trace_replay(
|
||||
_make_model_config(vocab_size=100), speculative_config=None
|
||||
)
|
||||
|
||||
|
||||
def test_trace_decode_token_ids_rejects_speculative_decoding():
|
||||
params = SamplingParams(trace_decode_token_ids=[1])
|
||||
with pytest.raises(ValueError, match="not supported with speculative decoding"):
|
||||
params._validate_trace_replay(
|
||||
_make_model_config(vocab_size=100), speculative_config=object()
|
||||
)
|
||||
|
||||
|
||||
def test_trace_decode_token_ids_rejects_structured_outputs():
|
||||
params = SamplingParams(
|
||||
trace_decode_token_ids=[1],
|
||||
structured_outputs=StructuredOutputsParams(json_object=True),
|
||||
)
|
||||
with pytest.raises(ValueError, match="not supported with structured outputs"):
|
||||
params._validate_trace_replay(
|
||||
_make_model_config(vocab_size=100), speculative_config=None
|
||||
)
|
||||
@@ -0,0 +1,195 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Model Runner V2 inference trace-replay.
|
||||
|
||||
Trace-replay forces the sampler to emit a predetermined sequence of decode
|
||||
token IDs. The replay step for a request is derived from GPU state as
|
||||
``total_len - prompt_len`` (output tokens produced so far), and the trace token
|
||||
overwrites the sampled token in place before logprobs are computed.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
pytest.importorskip("triton")
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required for trace-replay tests", allow_module_level=True)
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.worker.gpu.sample.trace_replay import (
|
||||
TraceReplayState,
|
||||
apply_trace_tokens,
|
||||
)
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
DEVICE = "cuda"
|
||||
TEST_MAX_MODEL_LEN = 2048
|
||||
|
||||
|
||||
def _i32(x) -> torch.Tensor:
|
||||
return torch.tensor(x, dtype=torch.int32, device=DEVICE)
|
||||
|
||||
|
||||
def _i64(x) -> torch.Tensor:
|
||||
return torch.tensor(x, dtype=torch.int64, device=DEVICE)
|
||||
|
||||
|
||||
def _trace_state(max_num_reqs: int) -> TraceReplayState:
|
||||
"""Build a state whose req_states exposes the buffers apply_trace reads.
|
||||
|
||||
total_len/prompt_len are written by the caller to position each request at
|
||||
the desired replay step.
|
||||
"""
|
||||
req_states = cast(
|
||||
RequestState,
|
||||
SimpleNamespace(
|
||||
max_num_reqs=max_num_reqs,
|
||||
max_model_len=TEST_MAX_MODEL_LEN,
|
||||
device=torch.device(DEVICE),
|
||||
total_len=SimpleNamespace(
|
||||
gpu=torch.zeros(max_num_reqs, dtype=torch.int32, device=DEVICE)
|
||||
),
|
||||
prompt_len=SimpleNamespace(
|
||||
gpu=torch.zeros(max_num_reqs, dtype=torch.int32, device=DEVICE)
|
||||
),
|
||||
),
|
||||
)
|
||||
return TraceReplayState(req_states)
|
||||
|
||||
|
||||
def _set_lens(state: TraceReplayState, total_len, prompt_len) -> None:
|
||||
"""Position each request at a replay step via total_len - prompt_len."""
|
||||
state.req_states.total_len.gpu[: len(total_len)] = _i32(total_len)
|
||||
state.req_states.prompt_len.gpu[: len(prompt_len)] = _i32(prompt_len)
|
||||
|
||||
|
||||
# ------------------------------ Kernel ------------------------------------
|
||||
|
||||
|
||||
def test_replay_overwrites_sampled_at_each_step():
|
||||
"""The trace token for the current step replaces the sampled token."""
|
||||
trace = [[100, 101, 102], [200, 201, 202]]
|
||||
trace_len = _i32([3, 3])
|
||||
trace_token_ids = torch.zeros(
|
||||
2, TEST_MAX_MODEL_LEN, dtype=torch.int32, device=DEVICE
|
||||
)
|
||||
for i, t in enumerate(trace):
|
||||
trace_token_ids[i, : len(t)] = _i32(t)
|
||||
prompt_len = _i32([5, 8])
|
||||
idx_mapping = _i32([0, 1])
|
||||
|
||||
for step in range(3):
|
||||
sampled = _i64([-7, -7]) # sentinel that must be overwritten
|
||||
total_len = _i32([5 + step, 8 + step])
|
||||
apply_trace_tokens(
|
||||
sampled, idx_mapping, trace_token_ids, trace_len, total_len, prompt_len
|
||||
)
|
||||
assert sampled.tolist() == [trace[0][step], trace[1][step]]
|
||||
|
||||
|
||||
def test_past_end_of_trace_leaves_sampled_untouched():
|
||||
"""Once step >= trace_len, the sampler's own token is kept."""
|
||||
trace_token_ids = torch.zeros(
|
||||
1, TEST_MAX_MODEL_LEN, dtype=torch.int32, device=DEVICE
|
||||
)
|
||||
trace_token_ids[0, :2] = _i32([100, 101])
|
||||
trace_len = _i32([2])
|
||||
prompt_len = _i32([4])
|
||||
idx_mapping = _i32([0])
|
||||
|
||||
sampled = _i64([999])
|
||||
total_len = _i32([4 + 2]) # step == 2 == trace_len -> out of range
|
||||
apply_trace_tokens(
|
||||
sampled, idx_mapping, trace_token_ids, trace_len, total_len, prompt_len
|
||||
)
|
||||
assert sampled.tolist() == [999]
|
||||
|
||||
|
||||
def test_non_trace_request_untouched():
|
||||
"""trace_len == 0 means the request never uses replay."""
|
||||
trace_token_ids = torch.zeros(
|
||||
1, TEST_MAX_MODEL_LEN, dtype=torch.int32, device=DEVICE
|
||||
)
|
||||
trace_len = _i32([0])
|
||||
prompt_len = _i32([3])
|
||||
idx_mapping = _i32([0])
|
||||
|
||||
sampled = _i64([42])
|
||||
total_len = _i32([3])
|
||||
apply_trace_tokens(
|
||||
sampled, idx_mapping, trace_token_ids, trace_len, total_len, prompt_len
|
||||
)
|
||||
assert sampled.tolist() == [42]
|
||||
|
||||
|
||||
def test_idx_mapping_indirection_and_negative_skip():
|
||||
"""batch_idx -> req_state_idx indirection, and negative entries are skipped."""
|
||||
# req_state 0: no trace; req_state 1: trace [500, 501].
|
||||
trace_token_ids = torch.zeros(
|
||||
2, TEST_MAX_MODEL_LEN, dtype=torch.int32, device=DEVICE
|
||||
)
|
||||
trace_token_ids[1, :2] = _i32([500, 501])
|
||||
trace_len = _i32([0, 2])
|
||||
prompt_len = _i32([10, 6])
|
||||
|
||||
# batch: [maps to state 1, masked (-1), maps to state 0].
|
||||
idx_mapping = _i32([1, -1, 0])
|
||||
sampled = _i64([1, 2, 3])
|
||||
# Indexed by req_state_idx: state 0 at total_len=10, state 1 at step 1.
|
||||
total_len = _i32([10, 6 + 1])
|
||||
apply_trace_tokens(
|
||||
sampled, idx_mapping, trace_token_ids, trace_len, total_len, prompt_len
|
||||
)
|
||||
# batch 0 -> state 1 step 1 -> 501; batch 1 masked; batch 2 -> state 0 no trace.
|
||||
assert sampled.tolist() == [501, 2, 3]
|
||||
|
||||
|
||||
# --------------------------- TraceReplayState -----------------------------
|
||||
|
||||
|
||||
def test_state_end_to_end():
|
||||
"""add_request -> apply_staged_writes -> apply_trace overwrites correctly."""
|
||||
state = _trace_state(4)
|
||||
state.add_request(0, SamplingParams(trace_decode_token_ids=[11, 22, 33]))
|
||||
state.add_request(1, SamplingParams()) # no trace
|
||||
state.apply_staged_writes()
|
||||
|
||||
idx_mapping = _i32([0, 1])
|
||||
sampled = _i64([-1, -1])
|
||||
_set_lens(state, [7 + 1, 7, 0, 0], [7, 7, 0, 0]) # req 0 at step 1
|
||||
state.apply_trace(sampled, idx_mapping)
|
||||
assert sampled.tolist() == [22, -1]
|
||||
|
||||
|
||||
def test_state_leaves_non_trace_batch_unchanged():
|
||||
"""Requests with trace_len == 0 remain unchanged after a trace was seen."""
|
||||
state = _trace_state(4)
|
||||
state.add_request(0, SamplingParams(trace_decode_token_ids=[11, 22]))
|
||||
state.add_request(1, SamplingParams())
|
||||
state.apply_staged_writes()
|
||||
|
||||
# Only the non-trace request (state 1) is in this batch.
|
||||
idx_mapping = _i32([1])
|
||||
sampled = _i64([555])
|
||||
_set_lens(state, [0, 7, 0, 0], [7, 7, 0, 0])
|
||||
state.apply_trace(sampled, idx_mapping)
|
||||
assert sampled.tolist() == [555]
|
||||
|
||||
|
||||
def test_slot_reuse_clears_trace():
|
||||
"""Reusing a slot for a non-trace request must not replay stale tokens."""
|
||||
state = _trace_state(2)
|
||||
state.add_request(0, SamplingParams(trace_decode_token_ids=[11, 22]))
|
||||
state.apply_staged_writes()
|
||||
# Slot 0 reused by a request without a trace.
|
||||
state.add_request(0, SamplingParams())
|
||||
state.apply_staged_writes()
|
||||
|
||||
idx_mapping = _i32([0])
|
||||
sampled = _i64([888])
|
||||
_set_lens(state, [3, 0], [3, 0])
|
||||
state.apply_trace(sampled, idx_mapping)
|
||||
assert sampled.tolist() == [888]
|
||||
Reference in New Issue
Block a user