[Model][Speculative Decoding] DeepSeek MTP spec decode (#12755)
Signed-off-by: Lu Fang <fanglu@fb.com> Co-authored-by: LiuXiaoxuanPKU <lilyliupku@gmail.com>
This commit is contained in:
co-authored by
LiuXiaoxuanPKU
parent
983a40a8bb
commit
f525c0be8b
@@ -99,6 +99,7 @@ class ModelInputForGPU(ModelRunnerInputBase):
|
||||
virtual_engine: int = 0
|
||||
async_callback: Optional[Callable] = None
|
||||
scheduler_outputs: Optional[SchedulerOutputs] = None
|
||||
previous_hidden_states: Optional[torch.Tensor] = None
|
||||
|
||||
def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
|
||||
tensor_dict = {
|
||||
@@ -1649,6 +1650,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
|
||||
kv_caches: List[torch.Tensor],
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
num_steps: int = 1,
|
||||
**kwargs,
|
||||
) -> Optional[Union[List[SamplerOutput], IntermediateTensors]]:
|
||||
if num_steps > 1:
|
||||
raise ValueError("num_steps > 1 is not supported in ModelRunner")
|
||||
@@ -1706,6 +1708,10 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
|
||||
"finished_requests_ids": model_input.finished_requests_ids,
|
||||
"request_ids_to_seq_ids": model_input.request_ids_to_seq_ids,
|
||||
} if self.has_inner_state else {}
|
||||
previous_hidden_states = kwargs.get("previous_hidden_states")
|
||||
model_kwargs = {}
|
||||
if previous_hidden_states is not None:
|
||||
model_kwargs["previous_hidden_states"] = previous_hidden_states
|
||||
if (self.observability_config is not None
|
||||
and self.observability_config.collect_model_forward_time):
|
||||
model_forward_start = torch.cuda.Event(enable_timing=True)
|
||||
@@ -1723,7 +1729,9 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
**MultiModalKwargs.as_kwargs(multi_modal_kwargs,
|
||||
device=self.device),
|
||||
**seqlen_agnostic_kwargs)
|
||||
**seqlen_agnostic_kwargs,
|
||||
**model_kwargs,
|
||||
)
|
||||
|
||||
if (self.observability_config is not None
|
||||
and self.observability_config.collect_model_forward_time):
|
||||
@@ -1815,7 +1823,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
|
||||
1. current vLLM instance is KV cache consumer/decode vLLM instance
|
||||
2. this batch is not a profiling run
|
||||
3. this batch is a prefill run
|
||||
|
||||
|
||||
Args:
|
||||
model_input: input to the model executable
|
||||
kv_caches: vLLM's paged memory
|
||||
@@ -1840,7 +1848,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
|
||||
1. current vLLM instance is KV cache producer/prefill vLLM instance
|
||||
2. this batch is not a profiling run
|
||||
3. this batch is a prefill run
|
||||
|
||||
|
||||
Args:
|
||||
model_input: input to the model executable
|
||||
kv_caches: vLLM's paged memory
|
||||
@@ -1976,7 +1984,11 @@ class CUDAGraphRunner(nn.Module):
|
||||
# Copy the input tensors to the input buffers.
|
||||
self.input_buffers["input_ids"].copy_(input_ids, non_blocking=True)
|
||||
if positions is not None:
|
||||
self.input_buffers["positions"].copy_(positions, non_blocking=True)
|
||||
# in some case like MLA, it will reuse positions in metadata
|
||||
# but truncate them to the original size
|
||||
# so the shape is not padded, we need to copy partial only
|
||||
self.input_buffers["positions"][:positions.shape[0]].copy_(
|
||||
positions, non_blocking=True)
|
||||
|
||||
if self.backend_name != "NO_ATTENTION":
|
||||
self.input_buffers["slot_mapping"].copy_(
|
||||
|
||||
@@ -46,7 +46,10 @@ def _init_attn_metadata_from_tensor_dict(
|
||||
valid_attn_kwargs = {}
|
||||
for field in dataclasses.fields(attn_backend.get_metadata_cls()):
|
||||
if field.name in tensor_dict:
|
||||
valid_attn_kwargs[field.name] = tensor_dict.pop(field.name)
|
||||
if field.name == "input_positions":
|
||||
valid_attn_kwargs[field.name] = tensor_dict[field.name]
|
||||
else:
|
||||
valid_attn_kwargs[field.name] = tensor_dict.pop(field.name)
|
||||
|
||||
attn_metadata = attn_backend.make_metadata(**valid_attn_kwargs)
|
||||
tensor_dict["attn_metadata"] = attn_metadata
|
||||
|
||||
@@ -68,10 +68,10 @@ class Worker(LocalOrDistributedWorkerBase):
|
||||
speculative_config = self.speculative_config
|
||||
model_config = self.model_config
|
||||
speculative_args = {} if speculative_config is None \
|
||||
or (speculative_config.draft_model_config.model ==
|
||||
model_config.model) \
|
||||
or (speculative_config.draft_model_config.hf_config.model_type ==
|
||||
model_config.hf_config.model_type) \
|
||||
or (speculative_config.draft_model_config.hf_config.model_type
|
||||
not in ["medusa", "mlp_speculator", "eagle"]) \
|
||||
not in ("medusa", "mlp_speculator", "eagle", "deepseek_mtp")) \
|
||||
else {"return_hidden_states": True}
|
||||
|
||||
ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
|
||||
|
||||
@@ -397,6 +397,8 @@ class LocalOrDistributedWorkerBase(WorkerBase):
|
||||
|
||||
model_input, worker_input, kwargs = inputs
|
||||
num_steps = worker_input.num_steps
|
||||
if (execute_model_req is not None and execute_model_req.spec_step_idx):
|
||||
kwargs["spec_step_idx"] = execute_model_req.spec_step_idx
|
||||
|
||||
self.execute_worker(worker_input)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user