[Core] NGram GPU Implementation compatible with Async Scheduler (#29184)
This commit is contained in:
@@ -10,7 +10,7 @@ from collections import defaultdict
|
||||
from collections.abc import Iterable, Iterator, Sequence
|
||||
from contextlib import contextmanager
|
||||
from copy import copy, deepcopy
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from functools import reduce
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, TypeAlias, cast
|
||||
|
||||
@@ -164,6 +164,12 @@ from vllm.v1.spec_decode.eagle import EagleProposer
|
||||
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
|
||||
from vllm.v1.spec_decode.medusa import MedusaProposer
|
||||
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
|
||||
from vllm.v1.spec_decode.ngram_proposer_gpu import (
|
||||
NgramProposerGPU,
|
||||
copy_num_valid_draft_tokens,
|
||||
update_ngram_gpu_tensors_incremental,
|
||||
update_scheduler_for_invalid_drafts,
|
||||
)
|
||||
from vllm.v1.spec_decode.suffix_decoding import SuffixDecodingProposer
|
||||
from vllm.v1.structured_output.utils import apply_grammar_bitmask
|
||||
from vllm.v1.utils import CpuGpuBuffer, record_function_or_nullcontext
|
||||
@@ -424,7 +430,7 @@ class GPUModelRunner(
|
||||
|
||||
# Broadcast PP output for external_launcher (torchrun)
|
||||
# to make sure we are synced across pp ranks
|
||||
# TODO: Support overlapping mirco-batches
|
||||
# TODO: Support overlapping micro-batches
|
||||
# https://github.com/vllm-project/vllm/issues/18019
|
||||
self.broadcast_pp_output = (
|
||||
self.parallel_config.distributed_executor_backend == "external_launcher"
|
||||
@@ -493,6 +499,7 @@ class GPUModelRunner(
|
||||
if self.speculative_config and get_pp_group().is_last_rank:
|
||||
self.drafter: (
|
||||
NgramProposer # noqa: F823
|
||||
| NgramProposerGPU
|
||||
| SuffixDecodingProposer
|
||||
| EagleProposer
|
||||
| DraftModelProposer
|
||||
@@ -509,6 +516,23 @@ class GPUModelRunner(
|
||||
device=self.device,
|
||||
runner=self,
|
||||
)
|
||||
elif self.speculative_config.use_ngram_gpu():
|
||||
self.drafter = NgramProposerGPU(self.vllm_config, self.device, self)
|
||||
self.num_tokens_no_spec_gpu = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.int32, device=device
|
||||
)
|
||||
self.token_ids_gpu_tensor = torch.zeros(
|
||||
self.max_num_reqs,
|
||||
self.max_model_len,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
self._ngram_pinned_idx_buf = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.long, pin_memory=True
|
||||
)
|
||||
self._ngram_pinned_val_buf = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.int32, pin_memory=True
|
||||
)
|
||||
elif self.speculative_config.method == "suffix":
|
||||
self.drafter = SuffixDecodingProposer(self.vllm_config)
|
||||
elif self.speculative_config.use_eagle():
|
||||
@@ -564,7 +588,7 @@ class GPUModelRunner(
|
||||
)
|
||||
self.input_batch = InputBatch(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
# We need to use the encoder length for encoder-decoer
|
||||
# We need to use the encoder length for encoder-decoder
|
||||
# because of KV cache for cross-attention.
|
||||
max_model_len=max(self.max_model_len, self.max_encoder_len),
|
||||
max_num_batched_tokens=self.max_num_tokens,
|
||||
@@ -721,6 +745,21 @@ class GPUModelRunner(
|
||||
|
||||
# Cached outputs.
|
||||
self._draft_token_ids: list[list[int]] | torch.Tensor | None = None
|
||||
# N-gram GPU path: async D2H buffer/event for per-request valid draft counts.
|
||||
self._num_valid_draft_tokens: torch.Tensor | None = None
|
||||
self._num_valid_draft_tokens_cpu: torch.Tensor | None = None
|
||||
self._num_valid_draft_tokens_event: torch.cuda.Event | None = None
|
||||
self._num_valid_draft_tokens_copy_stream: torch.cuda.Stream | None = None
|
||||
if (
|
||||
self.speculative_config is not None
|
||||
and self.speculative_config.use_ngram_gpu()
|
||||
):
|
||||
self._num_valid_draft_tokens_cpu = torch.empty(
|
||||
self.max_num_reqs, dtype=torch.int32, pin_memory=self.pin_memory
|
||||
)
|
||||
self._num_valid_draft_tokens_event = torch.cuda.Event()
|
||||
self._num_valid_draft_tokens_copy_stream = torch.cuda.Stream()
|
||||
|
||||
self._draft_token_req_ids: list[str] | None = None
|
||||
self.transfer_event = torch.Event()
|
||||
self.sampled_token_ids_pinned_cpu = torch.empty(
|
||||
@@ -992,6 +1031,13 @@ class GPUModelRunner(
|
||||
for req_id in unscheduled_req_ids:
|
||||
self.input_batch.remove_request(req_id)
|
||||
|
||||
is_ngram_gpu = (
|
||||
self.speculative_config is not None
|
||||
and self.speculative_config.use_ngram_gpu()
|
||||
)
|
||||
if is_ngram_gpu:
|
||||
ngram_gpu_new_reqs: list[CachedRequestState] = []
|
||||
|
||||
reqs_to_add: list[CachedRequestState] = []
|
||||
# Add new requests to the cached states.
|
||||
for new_req_data in scheduler_output.scheduled_new_reqs:
|
||||
@@ -1054,12 +1100,31 @@ class GPUModelRunner(
|
||||
self._init_xdrope_positions(req_state)
|
||||
|
||||
reqs_to_add.append(req_state)
|
||||
# Track new requests for ngram_gpu full tensor copy
|
||||
if is_ngram_gpu:
|
||||
ngram_gpu_new_reqs.append(req_state)
|
||||
|
||||
# Update the states of the running/resumed requests.
|
||||
is_last_rank = get_pp_group().is_last_rank
|
||||
req_data = scheduler_output.scheduled_cached_reqs
|
||||
scheduled_spec_tokens = scheduler_output.scheduled_spec_decode_tokens
|
||||
|
||||
# Save scheduler-allocated spec lengths before trimming so
|
||||
# prev_num_draft_len keeps the optimistic count for rejection correction.
|
||||
original_num_spec_per_req: dict[str, int] = {}
|
||||
if (
|
||||
self.speculative_config is not None
|
||||
and self.speculative_config.use_ngram_gpu()
|
||||
):
|
||||
for req_id, toks in scheduled_spec_tokens.items():
|
||||
original_num_spec_per_req[req_id] = len(toks)
|
||||
update_scheduler_for_invalid_drafts(
|
||||
self._num_valid_draft_tokens_event,
|
||||
self._num_valid_draft_tokens_cpu,
|
||||
scheduler_output,
|
||||
self.input_batch.req_id_to_index,
|
||||
)
|
||||
|
||||
# Wait until valid_sampled_tokens_count is copied to cpu,
|
||||
# then use it to update actual num_computed_tokens of each request.
|
||||
valid_sampled_token_count = self._get_valid_sampled_token_count()
|
||||
@@ -1076,13 +1141,13 @@ class GPUModelRunner(
|
||||
# prev_num_draft_len is used in async scheduling mode with
|
||||
# spec decode. it indicates if need to update num_computed_tokens
|
||||
# of the request. for example:
|
||||
# fist step: num_computed_tokens = 0, spec_tokens = [],
|
||||
# first step: num_computed_tokens = 0, spec_tokens = [],
|
||||
# prev_num_draft_len = 0.
|
||||
# second step: num_computed_tokens = 100(prompt length),
|
||||
# spec_tokens = [a,b], prev_num_draft_len = 0.
|
||||
# third step: num_computed_tokens = 100 + 2, spec_tokens = [c,d],
|
||||
# prev_num_draft_len = 2.
|
||||
# num_computed_tokens in first step and second step does't contain
|
||||
# num_computed_tokens in first step and second step doesn't contain
|
||||
# the spec tokens length, but in third step it contains the
|
||||
# spec tokens length. we only need to update num_computed_tokens
|
||||
# when prev_num_draft_len > 0.
|
||||
@@ -1096,6 +1161,9 @@ class GPUModelRunner(
|
||||
num_computed_tokens -= num_rejected
|
||||
req_state.output_token_ids.extend([-1] * num_accepted)
|
||||
|
||||
if is_ngram_gpu and num_accepted > 0 and req_index is not None:
|
||||
self.input_batch.num_tokens_no_spec[req_index] += num_accepted
|
||||
|
||||
# Update the cached states.
|
||||
req_state.num_computed_tokens = num_computed_tokens
|
||||
|
||||
@@ -1156,6 +1224,9 @@ class GPUModelRunner(
|
||||
req_state.output_token_ids = resumed_token_ids[-num_output_tokens:]
|
||||
|
||||
reqs_to_add.append(req_state)
|
||||
# Track resumed requests for ngram_gpu full tensor copy
|
||||
if is_ngram_gpu:
|
||||
ngram_gpu_new_reqs.append(req_state)
|
||||
continue
|
||||
|
||||
# Update the persistent batch.
|
||||
@@ -1176,6 +1247,11 @@ class GPUModelRunner(
|
||||
|
||||
# Add spec_token_ids to token_ids_cpu.
|
||||
self.input_batch.update_req_spec_token_ids(req_state, scheduled_spec_tokens)
|
||||
# Restore scheduler-side draft count after ngram trimming.
|
||||
if original_num_spec_per_req:
|
||||
orig = original_num_spec_per_req.get(req_id, 0)
|
||||
if orig != req_state.prev_num_draft_len:
|
||||
req_state.prev_num_draft_len = orig
|
||||
|
||||
# Add the new or resumed requests to the persistent batch.
|
||||
# The smaller empty indices are filled first.
|
||||
@@ -1190,6 +1266,18 @@ class GPUModelRunner(
|
||||
# Refresh batch metadata with any pending updates.
|
||||
self.input_batch.refresh_metadata()
|
||||
|
||||
# Incrementally update ngram_gpu tensors after batch is stable
|
||||
if is_ngram_gpu:
|
||||
update_ngram_gpu_tensors_incremental(
|
||||
self.input_batch,
|
||||
self.token_ids_gpu_tensor,
|
||||
self.num_tokens_no_spec_gpu,
|
||||
ngram_gpu_new_reqs,
|
||||
self.device,
|
||||
_pinned_idx_buf=self._ngram_pinned_idx_buf,
|
||||
_pinned_val_buf=self._ngram_pinned_val_buf,
|
||||
)
|
||||
|
||||
def _update_states_after_model_execute(
|
||||
self, output_token_ids: torch.Tensor, scheduler_output: "SchedulerOutput"
|
||||
) -> None:
|
||||
@@ -3412,6 +3500,23 @@ class GPUModelRunner(
|
||||
else:
|
||||
logger.error("RoutedExpertsCapturer not initialized.")
|
||||
|
||||
# If ngram_gpu is used, we need to copy the scheduler_output to avoid
|
||||
# the modification has influence on the scheduler_output in engine core process.
|
||||
# The replace is much faster than deepcopy.
|
||||
if (
|
||||
self.speculative_config is not None
|
||||
and self.speculative_config.use_ngram_gpu()
|
||||
):
|
||||
num_scheduled_tokens_copy = scheduler_output.num_scheduled_tokens.copy()
|
||||
spec_decode_tokens_copy = (
|
||||
scheduler_output.scheduled_spec_decode_tokens.copy()
|
||||
)
|
||||
scheduler_output = replace(
|
||||
scheduler_output,
|
||||
num_scheduled_tokens=num_scheduled_tokens_copy,
|
||||
scheduled_spec_decode_tokens=spec_decode_tokens_copy,
|
||||
)
|
||||
|
||||
if scheduler_output.preempted_req_ids and has_kv_transfer_group():
|
||||
get_kv_transfer_group().handle_preemptions(
|
||||
scheduler_output.preempted_req_ids
|
||||
@@ -3825,6 +3930,32 @@ class GPUModelRunner(
|
||||
self._copy_valid_sampled_token_count(
|
||||
next_token_ids, valid_sampled_tokens_count
|
||||
)
|
||||
self._draft_token_ids = torch.zeros(
|
||||
1, device=self.device, dtype=torch.int32
|
||||
).expand(len(self.input_batch.req_ids), self.num_spec_tokens)
|
||||
self._copy_draft_token_ids_to_cpu(scheduler_output, zeros_only=True)
|
||||
elif (
|
||||
spec_config.use_ngram_gpu()
|
||||
and not spec_config.disable_padded_drafter_batch
|
||||
):
|
||||
assert isinstance(self.drafter, NgramProposerGPU)
|
||||
sampled_token_ids = sampler_output.sampled_token_ids
|
||||
if input_fits_in_drafter:
|
||||
propose_draft_token_ids(sampled_token_ids)
|
||||
elif self.valid_sampled_token_count_event is not None:
|
||||
assert spec_decode_common_attn_metadata is not None
|
||||
next_token_ids, valid_sampled_tokens_count, _ = (
|
||||
self.drafter.update_token_ids_ngram(
|
||||
sampled_token_ids,
|
||||
self.input_batch,
|
||||
self.token_ids_gpu_tensor,
|
||||
self.num_tokens_no_spec_gpu,
|
||||
self.discard_request_mask.gpu,
|
||||
)
|
||||
)
|
||||
self._copy_valid_sampled_token_count(
|
||||
next_token_ids, valid_sampled_tokens_count
|
||||
)
|
||||
# Since we couldn't run the drafter,
|
||||
# just use zeros for the draft tokens.
|
||||
self._draft_token_ids = torch.zeros(
|
||||
@@ -4064,6 +4195,52 @@ class GPUModelRunner(
|
||||
self.input_batch.token_ids_cpu,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
if isinstance(self.drafter, NgramProposer):
|
||||
assert isinstance(sampled_token_ids, list), (
|
||||
"sampled_token_ids should be a python list when ngram is used."
|
||||
)
|
||||
draft_token_ids = self.drafter.propose(
|
||||
sampled_token_ids,
|
||||
self.input_batch.num_tokens_no_spec,
|
||||
self.input_batch.token_ids_cpu,
|
||||
)
|
||||
elif spec_config.use_ngram_gpu():
|
||||
assert isinstance(self.drafter, NgramProposerGPU)
|
||||
(
|
||||
next_token_ids,
|
||||
valid_sampled_tokens_count,
|
||||
valid_sampled_token_ids_gpu,
|
||||
) = self.drafter.update_token_ids_ngram(
|
||||
sampled_token_ids,
|
||||
self.input_batch,
|
||||
self.token_ids_gpu_tensor,
|
||||
self.num_tokens_no_spec_gpu,
|
||||
self.discard_request_mask.gpu,
|
||||
)
|
||||
self._copy_valid_sampled_token_count(
|
||||
next_token_ids, valid_sampled_tokens_count
|
||||
)
|
||||
|
||||
batch_size = next_token_ids.shape[0]
|
||||
|
||||
draft_token_ids, num_valid_draft_tokens = self.drafter.propose(
|
||||
self.num_tokens_no_spec_gpu[:batch_size],
|
||||
self.token_ids_gpu_tensor[:batch_size],
|
||||
valid_sampled_token_ids_gpu,
|
||||
valid_sampled_tokens_count,
|
||||
)
|
||||
|
||||
# Cache valid draft counts for scheduler-side trimming.
|
||||
self._num_valid_draft_tokens = num_valid_draft_tokens
|
||||
|
||||
# Async D2H copy on a dedicated stream.
|
||||
copy_num_valid_draft_tokens(
|
||||
self._num_valid_draft_tokens_cpu,
|
||||
self._num_valid_draft_tokens_copy_stream,
|
||||
self._num_valid_draft_tokens_event,
|
||||
self._num_valid_draft_tokens,
|
||||
self.input_batch.num_reqs,
|
||||
)
|
||||
elif spec_config.method == "suffix":
|
||||
assert isinstance(sampled_token_ids, list)
|
||||
assert isinstance(self.drafter, SuffixDecodingProposer)
|
||||
|
||||
Reference in New Issue
Block a user