[Feature] Mask Replay (#49577)

Signed-off-by: vx120 <893600387@qq.com>
Signed-off-by: vx120 <57470515+vx120@users.noreply.github.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
vx120
2026-08-13 08:44:31 +00:00
committed by GitHub
co-authored by aoshen02 Codex Claude Opus 4.6 OpenAI Codex
parent 10bcad2523
commit 50ba4bc6b2
24 changed files with 557 additions and 6 deletions
+13
View File
@@ -17,6 +17,7 @@ from vllm.outputs import (
PoolingOutput,
PoolingRequestOutput,
RequestOutput,
SamplingMask,
)
from vllm.sampling_params import RequestOutputKind
from vllm.tokenizers import TokenizerLike
@@ -37,6 +38,7 @@ from vllm.v1.metrics.stats import (
RequestStateStats,
SchedulerStats,
)
from vllm.v1.outputs import SamplingMaskLists
# shared empty CPU tensor used as a placeholder pooling output
EMPTY_CPU_TENSOR = torch.empty(0, device="cpu")
@@ -178,6 +180,7 @@ class RequestState:
# Routed experts accumulation (prompt + sample chunks)
self.routed_experts_chunks: list[np.ndarray] = []
self.sampling_mask_chunks: list[SamplingMaskLists] = []
# Stream Interval
self.stream_interval = stream_interval
@@ -406,6 +409,11 @@ class RequestState:
if delta and logprobs:
logprobs = logprobs[-len(token_ids) :]
sampling_mask = None
if finished and self.sampling_mask_chunks:
merged = SamplingMaskLists.merge(self.sampling_mask_chunks)
sampling_mask = SamplingMask(merged.to_nested_list())
# Concatenate routed experts on finish
routed_experts = None
if finished and self.routed_experts_chunks:
@@ -416,6 +424,7 @@ class RequestState:
text=text,
token_ids=token_ids,
routed_experts=routed_experts,
sampling_mask=sampling_mask,
logprobs=logprobs,
cumulative_logprob=self.logprobs_processor.cumulative_logprob,
finish_reason=str(finish_reason) if finished else None,
@@ -652,6 +661,10 @@ class OutputProcessor:
if pooling_output is None:
assert req_state.detokenizer is not None
assert req_state.logprobs_processor is not None
if engine_core_output.new_sampling_mask is not None:
req_state.sampling_mask_chunks.append(
engine_core_output.new_sampling_mask
)
# 2) Detokenize the token ids into text and perform stop checks.
stop_string = req_state.detokenizer.update(
new_token_ids, finish_reason == FinishReason.STOP