[Bugfix][Async] Fix async spec decoding with hybrid models (#38556)

Signed-off-by: SandishKumarHN <sandishkumarhn@gmail.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: SandishKumarHN <sandishkumarhn@gmail.com>
This commit is contained in:
Matthew Bonanni
2026-03-31 11:08:54 -04:00
committed by GitHub
co-authored by SandishKumarHN
parent 7337ff7f03
commit 757068dc65
6 changed files with 177 additions and 36 deletions
+22 -8
View File
@@ -210,7 +210,7 @@ from .utils import (
if TYPE_CHECKING:
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.spec_decode.ngram_proposer import NgramProposer
from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager
from vllm.v1.worker.gpu.mm.encoder_cudagraph import EncoderCudaGraphManager
logger = init_logger(__name__)
@@ -1939,9 +1939,24 @@ class GPUModelRunner(
# _update_states_after_model_execute for hybrid models).
if self.num_accepted_tokens_event is not None:
self.num_accepted_tokens_event.synchronize()
self.num_accepted_tokens.np[:num_reqs] = (
self.input_batch.num_accepted_tokens_cpu[:num_reqs]
)
# Async mode: condense() reordered indices, use prev_positions mapping
if self.use_async_scheduling and prev_req_id_to_index:
prev_idx = self.prev_positions.np[:num_reqs]
new_mask = prev_idx < 0
self.num_accepted_tokens.np[:num_reqs] = (
self.input_batch.num_accepted_tokens_cpu[
np.where(new_mask, 0, prev_idx)
]
)
self.num_accepted_tokens.np[:num_reqs][new_mask] = 1
self.input_batch.num_accepted_tokens_cpu[:num_reqs] = (
self.num_accepted_tokens.np[:num_reqs]
)
else:
# Non-async mode: use values directly
self.num_accepted_tokens.np[:num_reqs] = (
self.input_batch.num_accepted_tokens_cpu[:num_reqs]
)
self.num_accepted_tokens.np[num_reqs:].fill(1)
self.num_accepted_tokens.copy_to_gpu()
else:
@@ -4220,7 +4235,6 @@ class GPUModelRunner(
assert spec_decode_common_attn_metadata is not None
next_token_ids, valid_sampled_tokens_count = (
self.drafter.prepare_next_token_ids_padded(
self.optimistic_seq_lens_cpu,
sampled_token_ids,
self.requests,
self.input_batch,
@@ -4587,7 +4601,6 @@ class GPUModelRunner(
)
next_token_ids, valid_sampled_tokens_count = (
self.drafter.prepare_next_token_ids_padded(
self.optimistic_seq_lens_cpu,
sampled_token_ids,
self.requests,
self.input_batch,
@@ -4632,7 +4645,6 @@ class GPUModelRunner(
)
next_token_ids, valid_sampled_tokens_count = (
self.drafter.prepare_next_token_ids_padded(
self.optimistic_seq_lens_cpu,
sampled_token_ids,
self.requests,
self.input_batch,
@@ -5990,7 +6002,9 @@ class GPUModelRunner(
SupportsEncoderCudaGraph,
supports_encoder_cudagraph,
)
from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager
from vllm.v1.worker.gpu.mm.encoder_cudagraph import (
EncoderCudaGraphManager,
)
raw_model = self.get_model()
if supports_encoder_cudagraph(raw_model):