[Bugfix] Route every speculation-capable row through the speculative path so recurrent state stays consistent (#56531)

Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Vadim Gimpelson
2026-10-06 03:03:44 +04:00
committed by GitHub
co-authored by Cursor Agent mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 4a705c7b54
commit 0eac152707
8 changed files with 672 additions and 31 deletions
+11 -10
View File
@@ -277,18 +277,19 @@ class MambaHybridModelState(DefaultModelState):
]
# GDN uses >= 0 to select spec-decode rows, so non-decode rows
# need the -1 sentinel rather than a raw zero draft count.
# need the -1 sentinel. Decode rows without drafts stay spec rows:
# only that path applies the previous step's accepted offset.
num_decode_draft_tokens_np = np.full(num_reqs, -1, dtype=np.int32)
num_draft_tokens_per_req = input_batch.num_draft_tokens_per_req
if num_draft_tokens_per_req is not None:
# Test request state, not num_scheduled_tokens == draft_count+1:
# adaptive rewrites num_scheduled_tokens to an even split, so that
# equality rarely holds and would demote every verify row to decode.
is_decode = ~is_prefilling_np & (input_batch.num_scheduled_tokens > 0)
spec_decode_mask = (num_draft_tokens_per_req > 0) & is_decode
num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where(
spec_decode_mask, num_draft_tokens_per_req, -1
)
if num_draft_tokens_per_req is None:
num_draft_tokens_per_req = np.zeros(input_batch.num_reqs, np.int32)
# Test request state, not num_scheduled_tokens == draft_count+1:
# adaptive rewrites num_scheduled_tokens to an even split, so that
# equality rarely holds and would demote every verify row to decode.
is_decode = ~is_prefilling_np & (input_batch.num_scheduled_tokens > 0)
num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where(
is_decode, num_draft_tokens_per_req, -1
)
num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np)
if self._align_mode: