[Model Runner V2][Spec Decode] Add multi-layer MTP speculator (#48892)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -223,6 +223,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.is_pooling_model = self.model_config.runner_type == "pooling"
|
||||
self.pooling_runner: PoolingRunner | None = None
|
||||
|
||||
# Multi-module MTP feeds its modules the next num_speculative_steps prefill
|
||||
# tokens during chunked prefill. Other speculators only read the immediate
|
||||
# next one.
|
||||
num_prefill_lookahead = (
|
||||
self.num_speculative_steps
|
||||
if self.speculative_config is not None
|
||||
and self.speculative_config.use_multi_module_mtp()
|
||||
else 1
|
||||
)
|
||||
# General request states.
|
||||
self.req_states = RequestState(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
@@ -231,6 +240,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_speculative_steps=self.num_speculative_steps,
|
||||
vocab_size=self.vocab_size,
|
||||
device=self.device,
|
||||
num_prefill_lookahead=num_prefill_lookahead,
|
||||
)
|
||||
self.input_buffers = InputBuffers(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
@@ -628,7 +638,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
torch.zeros(
|
||||
input_batch.num_tokens,
|
||||
dtype=torch.bool,
|
||||
device=self.device,
|
||||
device="cpu",
|
||||
),
|
||||
)
|
||||
|
||||
@@ -1521,6 +1531,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# NOTE: This is done here because postprocess updates
|
||||
# num_computed_prefill_tokens.
|
||||
# The EAGLE/MTP drafter reads one position ahead of the target.
|
||||
# TODO(TheEpicDolphin): Gather MM embeddings for all speculative
|
||||
# steps during multi-module MTP.
|
||||
mm_inputs = self.model_state.gather_mm_embeddings(
|
||||
input_batch, draft_lookahead=1
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user