[ModelRunner V2] Fix Mamba2 crash on non-spec-decode (#47428)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -246,7 +246,7 @@ class MambaHybridModelState(DefaultModelState):
|
|||||||
# compute them during actual (non-capture) forward execution.
|
# compute them during actual (non-capture) forward execution.
|
||||||
num_accepted_tokens = None
|
num_accepted_tokens = None
|
||||||
num_decode_draft_tokens_cpu = None
|
num_decode_draft_tokens_cpu = None
|
||||||
if not for_capture:
|
if not for_capture and self.vllm_config.num_speculative_tokens > 0:
|
||||||
num_accepted_tokens = self.num_accepted_tokens_gpu.new_ones(num_reqs)
|
num_accepted_tokens = self.num_accepted_tokens_gpu.new_ones(num_reqs)
|
||||||
num_accepted_tokens[: input_batch.num_reqs] = self.num_accepted_tokens_gpu[
|
num_accepted_tokens[: input_batch.num_reqs] = self.num_accepted_tokens_gpu[
|
||||||
input_batch.idx_mapping
|
input_batch.idx_mapping
|
||||||
|
|||||||
Reference in New Issue
Block a user