[Feature] Mask Replay (#49577)
Signed-off-by: vx120 <893600387@qq.com> Signed-off-by: vx120 <57470515+vx120@users.noreply.github.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
aoshen02
Codex
Claude Opus 4.6
OpenAI Codex
parent
10bcad2523
commit
50ba4bc6b2
@@ -17,6 +17,7 @@ from vllm.outputs import (
|
||||
PoolingOutput,
|
||||
PoolingRequestOutput,
|
||||
RequestOutput,
|
||||
SamplingMask,
|
||||
)
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
@@ -37,6 +38,7 @@ from vllm.v1.metrics.stats import (
|
||||
RequestStateStats,
|
||||
SchedulerStats,
|
||||
)
|
||||
from vllm.v1.outputs import SamplingMaskLists
|
||||
|
||||
# shared empty CPU tensor used as a placeholder pooling output
|
||||
EMPTY_CPU_TENSOR = torch.empty(0, device="cpu")
|
||||
@@ -178,6 +180,7 @@ class RequestState:
|
||||
|
||||
# Routed experts accumulation (prompt + sample chunks)
|
||||
self.routed_experts_chunks: list[np.ndarray] = []
|
||||
self.sampling_mask_chunks: list[SamplingMaskLists] = []
|
||||
|
||||
# Stream Interval
|
||||
self.stream_interval = stream_interval
|
||||
@@ -406,6 +409,11 @@ class RequestState:
|
||||
if delta and logprobs:
|
||||
logprobs = logprobs[-len(token_ids) :]
|
||||
|
||||
sampling_mask = None
|
||||
if finished and self.sampling_mask_chunks:
|
||||
merged = SamplingMaskLists.merge(self.sampling_mask_chunks)
|
||||
sampling_mask = SamplingMask(merged.to_nested_list())
|
||||
|
||||
# Concatenate routed experts on finish
|
||||
routed_experts = None
|
||||
if finished and self.routed_experts_chunks:
|
||||
@@ -416,6 +424,7 @@ class RequestState:
|
||||
text=text,
|
||||
token_ids=token_ids,
|
||||
routed_experts=routed_experts,
|
||||
sampling_mask=sampling_mask,
|
||||
logprobs=logprobs,
|
||||
cumulative_logprob=self.logprobs_processor.cumulative_logprob,
|
||||
finish_reason=str(finish_reason) if finished else None,
|
||||
@@ -652,6 +661,10 @@ class OutputProcessor:
|
||||
if pooling_output is None:
|
||||
assert req_state.detokenizer is not None
|
||||
assert req_state.logprobs_processor is not None
|
||||
if engine_core_output.new_sampling_mask is not None:
|
||||
req_state.sampling_mask_chunks.append(
|
||||
engine_core_output.new_sampling_mask
|
||||
)
|
||||
# 2) Detokenize the token ids into text and perform stop checks.
|
||||
stop_string = req_state.detokenizer.update(
|
||||
new_token_ids, finish_reason == FinishReason.STOP
|
||||
|
||||
Reference in New Issue
Block a user