[Bugfix][Async] Fix async spec decoding with hybrid models (#38556)
Signed-off-by: SandishKumarHN <sandishkumarhn@gmail.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: SandishKumarHN <sandishkumarhn@gmail.com>
This commit is contained in:
co-authored by
SandishKumarHN
parent
7337ff7f03
commit
757068dc65
@@ -210,7 +210,7 @@ from .utils import (
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.spec_decode.ngram_proposer import NgramProposer
|
||||
from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager
|
||||
from vllm.v1.worker.gpu.mm.encoder_cudagraph import EncoderCudaGraphManager
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -1939,9 +1939,24 @@ class GPUModelRunner(
|
||||
# _update_states_after_model_execute for hybrid models).
|
||||
if self.num_accepted_tokens_event is not None:
|
||||
self.num_accepted_tokens_event.synchronize()
|
||||
self.num_accepted_tokens.np[:num_reqs] = (
|
||||
self.input_batch.num_accepted_tokens_cpu[:num_reqs]
|
||||
)
|
||||
# Async mode: condense() reordered indices, use prev_positions mapping
|
||||
if self.use_async_scheduling and prev_req_id_to_index:
|
||||
prev_idx = self.prev_positions.np[:num_reqs]
|
||||
new_mask = prev_idx < 0
|
||||
self.num_accepted_tokens.np[:num_reqs] = (
|
||||
self.input_batch.num_accepted_tokens_cpu[
|
||||
np.where(new_mask, 0, prev_idx)
|
||||
]
|
||||
)
|
||||
self.num_accepted_tokens.np[:num_reqs][new_mask] = 1
|
||||
self.input_batch.num_accepted_tokens_cpu[:num_reqs] = (
|
||||
self.num_accepted_tokens.np[:num_reqs]
|
||||
)
|
||||
else:
|
||||
# Non-async mode: use values directly
|
||||
self.num_accepted_tokens.np[:num_reqs] = (
|
||||
self.input_batch.num_accepted_tokens_cpu[:num_reqs]
|
||||
)
|
||||
self.num_accepted_tokens.np[num_reqs:].fill(1)
|
||||
self.num_accepted_tokens.copy_to_gpu()
|
||||
else:
|
||||
@@ -4220,7 +4235,6 @@ class GPUModelRunner(
|
||||
assert spec_decode_common_attn_metadata is not None
|
||||
next_token_ids, valid_sampled_tokens_count = (
|
||||
self.drafter.prepare_next_token_ids_padded(
|
||||
self.optimistic_seq_lens_cpu,
|
||||
sampled_token_ids,
|
||||
self.requests,
|
||||
self.input_batch,
|
||||
@@ -4587,7 +4601,6 @@ class GPUModelRunner(
|
||||
)
|
||||
next_token_ids, valid_sampled_tokens_count = (
|
||||
self.drafter.prepare_next_token_ids_padded(
|
||||
self.optimistic_seq_lens_cpu,
|
||||
sampled_token_ids,
|
||||
self.requests,
|
||||
self.input_batch,
|
||||
@@ -4632,7 +4645,6 @@ class GPUModelRunner(
|
||||
)
|
||||
next_token_ids, valid_sampled_tokens_count = (
|
||||
self.drafter.prepare_next_token_ids_padded(
|
||||
self.optimistic_seq_lens_cpu,
|
||||
sampled_token_ids,
|
||||
self.requests,
|
||||
self.input_batch,
|
||||
@@ -5990,7 +6002,9 @@ class GPUModelRunner(
|
||||
SupportsEncoderCudaGraph,
|
||||
supports_encoder_cudagraph,
|
||||
)
|
||||
from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager
|
||||
from vllm.v1.worker.gpu.mm.encoder_cudagraph import (
|
||||
EncoderCudaGraphManager,
|
||||
)
|
||||
|
||||
raw_model = self.get_model()
|
||||
if supports_encoder_cudagraph(raw_model):
|
||||
|
||||
Reference in New Issue
Block a user