[Bugfix] Preserve sampling masks in DELTA and TITO streaming outputs (#55935)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: fh3 <a@b> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: fh3 <a@b> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
fh3
Claude Fable 5.1
parent
c9578f10ab
commit
4a9d6f9572
@@ -439,7 +439,7 @@ class RequestState:
|
||||
logprobs = logprobs[-num_new_tokens:] if num_new_tokens else logprobs[:0]
|
||||
|
||||
sampling_mask = None
|
||||
if finished and self.sampling_mask_chunks:
|
||||
if (delta or finished) and self.sampling_mask_chunks:
|
||||
sampling_mask = SamplingMask(
|
||||
[
|
||||
position
|
||||
@@ -447,6 +447,8 @@ class RequestState:
|
||||
for position in chunk.to_nested_list()
|
||||
]
|
||||
)
|
||||
if delta:
|
||||
self.sampling_mask_chunks.clear()
|
||||
|
||||
# Concatenate routed experts on finish
|
||||
routed_experts = None
|
||||
|
||||
Reference in New Issue
Block a user