[Bugfix] Route every speculation-capable row through the speculative path so recurrent state stays consistent (#56531)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com> Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor Agent
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4a705c7b54
commit
0eac152707
@@ -277,18 +277,19 @@ class MambaHybridModelState(DefaultModelState):
|
||||
]
|
||||
|
||||
# GDN uses >= 0 to select spec-decode rows, so non-decode rows
|
||||
# need the -1 sentinel rather than a raw zero draft count.
|
||||
# need the -1 sentinel. Decode rows without drafts stay spec rows:
|
||||
# only that path applies the previous step's accepted offset.
|
||||
num_decode_draft_tokens_np = np.full(num_reqs, -1, dtype=np.int32)
|
||||
num_draft_tokens_per_req = input_batch.num_draft_tokens_per_req
|
||||
if num_draft_tokens_per_req is not None:
|
||||
# Test request state, not num_scheduled_tokens == draft_count+1:
|
||||
# adaptive rewrites num_scheduled_tokens to an even split, so that
|
||||
# equality rarely holds and would demote every verify row to decode.
|
||||
is_decode = ~is_prefilling_np & (input_batch.num_scheduled_tokens > 0)
|
||||
spec_decode_mask = (num_draft_tokens_per_req > 0) & is_decode
|
||||
num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where(
|
||||
spec_decode_mask, num_draft_tokens_per_req, -1
|
||||
)
|
||||
if num_draft_tokens_per_req is None:
|
||||
num_draft_tokens_per_req = np.zeros(input_batch.num_reqs, np.int32)
|
||||
# Test request state, not num_scheduled_tokens == draft_count+1:
|
||||
# adaptive rewrites num_scheduled_tokens to an even split, so that
|
||||
# equality rarely holds and would demote every verify row to decode.
|
||||
is_decode = ~is_prefilling_np & (input_batch.num_scheduled_tokens > 0)
|
||||
num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where(
|
||||
is_decode, num_draft_tokens_per_req, -1
|
||||
)
|
||||
num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np)
|
||||
|
||||
if self._align_mode:
|
||||
|
||||
Reference in New Issue
Block a user