[Core] NGram GPU Implementation compatible with Async Scheduler (#29184)

This commit is contained in:
PatchyTIS
2026-03-07 13:51:37 -08:00
committed by GitHub
parent ee54f9cdb9
commit a6be75dbd2
9 changed files with 940 additions and 12 deletions
+182 -5
View File
@@ -10,7 +10,7 @@ from collections import defaultdict
from collections.abc import Iterable, Iterator, Sequence
from contextlib import contextmanager
from copy import copy, deepcopy
from dataclasses import dataclass
from dataclasses import dataclass, replace
from functools import reduce
from typing import TYPE_CHECKING, Any, NamedTuple, TypeAlias, cast
@@ -164,6 +164,12 @@ from vllm.v1.spec_decode.eagle import EagleProposer
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
from vllm.v1.spec_decode.medusa import MedusaProposer
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
from vllm.v1.spec_decode.ngram_proposer_gpu import (
NgramProposerGPU,
copy_num_valid_draft_tokens,
update_ngram_gpu_tensors_incremental,
update_scheduler_for_invalid_drafts,
)
from vllm.v1.spec_decode.suffix_decoding import SuffixDecodingProposer
from vllm.v1.structured_output.utils import apply_grammar_bitmask
from vllm.v1.utils import CpuGpuBuffer, record_function_or_nullcontext
@@ -424,7 +430,7 @@ class GPUModelRunner(
# Broadcast PP output for external_launcher (torchrun)
# to make sure we are synced across pp ranks
# TODO: Support overlapping mirco-batches
# TODO: Support overlapping micro-batches
# https://github.com/vllm-project/vllm/issues/18019
self.broadcast_pp_output = (
self.parallel_config.distributed_executor_backend == "external_launcher"
@@ -493,6 +499,7 @@ class GPUModelRunner(
if self.speculative_config and get_pp_group().is_last_rank:
self.drafter: (
NgramProposer # noqa: F823
| NgramProposerGPU
| SuffixDecodingProposer
| EagleProposer
| DraftModelProposer
@@ -509,6 +516,23 @@ class GPUModelRunner(
device=self.device,
runner=self,
)
elif self.speculative_config.use_ngram_gpu():
self.drafter = NgramProposerGPU(self.vllm_config, self.device, self)
self.num_tokens_no_spec_gpu = torch.zeros(
self.max_num_reqs, dtype=torch.int32, device=device
)
self.token_ids_gpu_tensor = torch.zeros(
self.max_num_reqs,
self.max_model_len,
dtype=torch.int32,
device=device,
)
self._ngram_pinned_idx_buf = torch.zeros(
self.max_num_reqs, dtype=torch.long, pin_memory=True
)
self._ngram_pinned_val_buf = torch.zeros(
self.max_num_reqs, dtype=torch.int32, pin_memory=True
)
elif self.speculative_config.method == "suffix":
self.drafter = SuffixDecodingProposer(self.vllm_config)
elif self.speculative_config.use_eagle():
@@ -564,7 +588,7 @@ class GPUModelRunner(
)
self.input_batch = InputBatch(
max_num_reqs=self.max_num_reqs,
# We need to use the encoder length for encoder-decoer
# We need to use the encoder length for encoder-decoder
# because of KV cache for cross-attention.
max_model_len=max(self.max_model_len, self.max_encoder_len),
max_num_batched_tokens=self.max_num_tokens,
@@ -721,6 +745,21 @@ class GPUModelRunner(
# Cached outputs.
self._draft_token_ids: list[list[int]] | torch.Tensor | None = None
# N-gram GPU path: async D2H buffer/event for per-request valid draft counts.
self._num_valid_draft_tokens: torch.Tensor | None = None
self._num_valid_draft_tokens_cpu: torch.Tensor | None = None
self._num_valid_draft_tokens_event: torch.cuda.Event | None = None
self._num_valid_draft_tokens_copy_stream: torch.cuda.Stream | None = None
if (
self.speculative_config is not None
and self.speculative_config.use_ngram_gpu()
):
self._num_valid_draft_tokens_cpu = torch.empty(
self.max_num_reqs, dtype=torch.int32, pin_memory=self.pin_memory
)
self._num_valid_draft_tokens_event = torch.cuda.Event()
self._num_valid_draft_tokens_copy_stream = torch.cuda.Stream()
self._draft_token_req_ids: list[str] | None = None
self.transfer_event = torch.Event()
self.sampled_token_ids_pinned_cpu = torch.empty(
@@ -992,6 +1031,13 @@ class GPUModelRunner(
for req_id in unscheduled_req_ids:
self.input_batch.remove_request(req_id)
is_ngram_gpu = (
self.speculative_config is not None
and self.speculative_config.use_ngram_gpu()
)
if is_ngram_gpu:
ngram_gpu_new_reqs: list[CachedRequestState] = []
reqs_to_add: list[CachedRequestState] = []
# Add new requests to the cached states.
for new_req_data in scheduler_output.scheduled_new_reqs:
@@ -1054,12 +1100,31 @@ class GPUModelRunner(
self._init_xdrope_positions(req_state)
reqs_to_add.append(req_state)
# Track new requests for ngram_gpu full tensor copy
if is_ngram_gpu:
ngram_gpu_new_reqs.append(req_state)
# Update the states of the running/resumed requests.
is_last_rank = get_pp_group().is_last_rank
req_data = scheduler_output.scheduled_cached_reqs
scheduled_spec_tokens = scheduler_output.scheduled_spec_decode_tokens
# Save scheduler-allocated spec lengths before trimming so
# prev_num_draft_len keeps the optimistic count for rejection correction.
original_num_spec_per_req: dict[str, int] = {}
if (
self.speculative_config is not None
and self.speculative_config.use_ngram_gpu()
):
for req_id, toks in scheduled_spec_tokens.items():
original_num_spec_per_req[req_id] = len(toks)
update_scheduler_for_invalid_drafts(
self._num_valid_draft_tokens_event,
self._num_valid_draft_tokens_cpu,
scheduler_output,
self.input_batch.req_id_to_index,
)
# Wait until valid_sampled_tokens_count is copied to cpu,
# then use it to update actual num_computed_tokens of each request.
valid_sampled_token_count = self._get_valid_sampled_token_count()
@@ -1076,13 +1141,13 @@ class GPUModelRunner(
# prev_num_draft_len is used in async scheduling mode with
# spec decode. it indicates if need to update num_computed_tokens
# of the request. for example:
# fist step: num_computed_tokens = 0, spec_tokens = [],
# first step: num_computed_tokens = 0, spec_tokens = [],
# prev_num_draft_len = 0.
# second step: num_computed_tokens = 100(prompt length),
# spec_tokens = [a,b], prev_num_draft_len = 0.
# third step: num_computed_tokens = 100 + 2, spec_tokens = [c,d],
# prev_num_draft_len = 2.
# num_computed_tokens in first step and second step does't contain
# num_computed_tokens in first step and second step doesn't contain
# the spec tokens length, but in third step it contains the
# spec tokens length. we only need to update num_computed_tokens
# when prev_num_draft_len > 0.
@@ -1096,6 +1161,9 @@ class GPUModelRunner(
num_computed_tokens -= num_rejected
req_state.output_token_ids.extend([-1] * num_accepted)
if is_ngram_gpu and num_accepted > 0 and req_index is not None:
self.input_batch.num_tokens_no_spec[req_index] += num_accepted
# Update the cached states.
req_state.num_computed_tokens = num_computed_tokens
@@ -1156,6 +1224,9 @@ class GPUModelRunner(
req_state.output_token_ids = resumed_token_ids[-num_output_tokens:]
reqs_to_add.append(req_state)
# Track resumed requests for ngram_gpu full tensor copy
if is_ngram_gpu:
ngram_gpu_new_reqs.append(req_state)
continue
# Update the persistent batch.
@@ -1176,6 +1247,11 @@ class GPUModelRunner(
# Add spec_token_ids to token_ids_cpu.
self.input_batch.update_req_spec_token_ids(req_state, scheduled_spec_tokens)
# Restore scheduler-side draft count after ngram trimming.
if original_num_spec_per_req:
orig = original_num_spec_per_req.get(req_id, 0)
if orig != req_state.prev_num_draft_len:
req_state.prev_num_draft_len = orig
# Add the new or resumed requests to the persistent batch.
# The smaller empty indices are filled first.
@@ -1190,6 +1266,18 @@ class GPUModelRunner(
# Refresh batch metadata with any pending updates.
self.input_batch.refresh_metadata()
# Incrementally update ngram_gpu tensors after batch is stable
if is_ngram_gpu:
update_ngram_gpu_tensors_incremental(
self.input_batch,
self.token_ids_gpu_tensor,
self.num_tokens_no_spec_gpu,
ngram_gpu_new_reqs,
self.device,
_pinned_idx_buf=self._ngram_pinned_idx_buf,
_pinned_val_buf=self._ngram_pinned_val_buf,
)
def _update_states_after_model_execute(
self, output_token_ids: torch.Tensor, scheduler_output: "SchedulerOutput"
) -> None:
@@ -3412,6 +3500,23 @@ class GPUModelRunner(
else:
logger.error("RoutedExpertsCapturer not initialized.")
# If ngram_gpu is used, we need to copy the scheduler_output to avoid
# the modification has influence on the scheduler_output in engine core process.
# The replace is much faster than deepcopy.
if (
self.speculative_config is not None
and self.speculative_config.use_ngram_gpu()
):
num_scheduled_tokens_copy = scheduler_output.num_scheduled_tokens.copy()
spec_decode_tokens_copy = (
scheduler_output.scheduled_spec_decode_tokens.copy()
)
scheduler_output = replace(
scheduler_output,
num_scheduled_tokens=num_scheduled_tokens_copy,
scheduled_spec_decode_tokens=spec_decode_tokens_copy,
)
if scheduler_output.preempted_req_ids and has_kv_transfer_group():
get_kv_transfer_group().handle_preemptions(
scheduler_output.preempted_req_ids
@@ -3825,6 +3930,32 @@ class GPUModelRunner(
self._copy_valid_sampled_token_count(
next_token_ids, valid_sampled_tokens_count
)
self._draft_token_ids = torch.zeros(
1, device=self.device, dtype=torch.int32
).expand(len(self.input_batch.req_ids), self.num_spec_tokens)
self._copy_draft_token_ids_to_cpu(scheduler_output, zeros_only=True)
elif (
spec_config.use_ngram_gpu()
and not spec_config.disable_padded_drafter_batch
):
assert isinstance(self.drafter, NgramProposerGPU)
sampled_token_ids = sampler_output.sampled_token_ids
if input_fits_in_drafter:
propose_draft_token_ids(sampled_token_ids)
elif self.valid_sampled_token_count_event is not None:
assert spec_decode_common_attn_metadata is not None
next_token_ids, valid_sampled_tokens_count, _ = (
self.drafter.update_token_ids_ngram(
sampled_token_ids,
self.input_batch,
self.token_ids_gpu_tensor,
self.num_tokens_no_spec_gpu,
self.discard_request_mask.gpu,
)
)
self._copy_valid_sampled_token_count(
next_token_ids, valid_sampled_tokens_count
)
# Since we couldn't run the drafter,
# just use zeros for the draft tokens.
self._draft_token_ids = torch.zeros(
@@ -4064,6 +4195,52 @@ class GPUModelRunner(
self.input_batch.token_ids_cpu,
slot_mappings=slot_mappings,
)
if isinstance(self.drafter, NgramProposer):
assert isinstance(sampled_token_ids, list), (
"sampled_token_ids should be a python list when ngram is used."
)
draft_token_ids = self.drafter.propose(
sampled_token_ids,
self.input_batch.num_tokens_no_spec,
self.input_batch.token_ids_cpu,
)
elif spec_config.use_ngram_gpu():
assert isinstance(self.drafter, NgramProposerGPU)
(
next_token_ids,
valid_sampled_tokens_count,
valid_sampled_token_ids_gpu,
) = self.drafter.update_token_ids_ngram(
sampled_token_ids,
self.input_batch,
self.token_ids_gpu_tensor,
self.num_tokens_no_spec_gpu,
self.discard_request_mask.gpu,
)
self._copy_valid_sampled_token_count(
next_token_ids, valid_sampled_tokens_count
)
batch_size = next_token_ids.shape[0]
draft_token_ids, num_valid_draft_tokens = self.drafter.propose(
self.num_tokens_no_spec_gpu[:batch_size],
self.token_ids_gpu_tensor[:batch_size],
valid_sampled_token_ids_gpu,
valid_sampled_tokens_count,
)
# Cache valid draft counts for scheduler-side trimming.
self._num_valid_draft_tokens = num_valid_draft_tokens
# Async D2H copy on a dedicated stream.
copy_num_valid_draft_tokens(
self._num_valid_draft_tokens_cpu,
self._num_valid_draft_tokens_copy_stream,
self._num_valid_draft_tokens_event,
self._num_valid_draft_tokens,
self.input_batch.num_reqs,
)
elif spec_config.method == "suffix":
assert isinstance(sampled_token_ids, list)
assert isinstance(self.drafter, SuffixDecodingProposer)