[Spec Decode] DSpark (#46995)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Giancarlo Delfin
mgoin
parent
63fcce4de1
commit
f5a8d73377
@@ -54,6 +54,7 @@ MTPModelTypes = Literal[
|
||||
]
|
||||
NgramGPUTypes = Literal["ngram_gpu"]
|
||||
DFlashModelTypes = Literal["dflash"]
|
||||
DSparkModelTypes = Literal["dspark"]
|
||||
EagleModelTypes = Literal[
|
||||
"eagle", "eagle3", "extract_hidden_states", MTPModelTypes, DFlashModelTypes
|
||||
]
|
||||
@@ -66,6 +67,7 @@ SpeculativeMethod = Literal[
|
||||
"custom_class",
|
||||
EagleModelTypes,
|
||||
NgramGPUTypes,
|
||||
DSparkModelTypes,
|
||||
]
|
||||
RejectionSampleMethod = Literal["standard", "synthetic", "block"]
|
||||
DraftSampleMethod = Literal["greedy", "probabilistic"]
|
||||
@@ -291,6 +293,7 @@ class SpeculativeConfig:
|
||||
"eagle3",
|
||||
"extract_hidden_states",
|
||||
"dflash",
|
||||
"dspark",
|
||||
)
|
||||
factors.append(uses_aux_hidden_states)
|
||||
|
||||
@@ -608,6 +611,13 @@ class SpeculativeConfig:
|
||||
# --quantization fp8 with a bf16 checkpoint.
|
||||
if not self.quantization:
|
||||
self.quantization = self.target_model_config.quantization
|
||||
elif self.method == "dspark":
|
||||
# DeepSeek DSpark can ship the weights inside the target checkpoint
|
||||
if self.target_model_config is None:
|
||||
raise ValueError("target_model_config must be present for dspark")
|
||||
self.model = self.target_model_config.model
|
||||
if not self.quantization:
|
||||
self.quantization = self.target_model_config.quantization
|
||||
elif self.method in ("ngram", "[ngram]"):
|
||||
self.model = "ngram"
|
||||
elif self.method == "ngram_gpu":
|
||||
@@ -755,18 +765,24 @@ class SpeculativeConfig:
|
||||
draft_hf.truncated_vocab_size = target_vocab
|
||||
|
||||
# Automatically detect the method
|
||||
if self.method in ("eagle", "eagle3", "dflash"):
|
||||
if self.method in ("eagle", "eagle3", "dflash", "dspark"):
|
||||
pass
|
||||
# examples:
|
||||
# yuhuili/EAGLE-LLaMA3-Instruct-8B
|
||||
# yuhuili/EAGLE3-LLaMA3.1-Instruct-8B
|
||||
# AngelSlim/Qwen3-8B_eagle3
|
||||
# deepseek-ai/dspark_qwen3_8b_block7
|
||||
elif "eagle-" in self.draft_model_config.model.lower():
|
||||
self.method = "eagle"
|
||||
elif "eagle3" in self.draft_model_config.model.lower():
|
||||
self.method = "eagle3"
|
||||
elif "dflash" in self.draft_model_config.model.lower():
|
||||
self.method = "dflash"
|
||||
elif (
|
||||
"dspark" in self.draft_model_config.model.lower()
|
||||
or "Qwen3DSparkModel" in self.draft_model_config.architectures
|
||||
):
|
||||
self.method = "dspark"
|
||||
elif self.draft_model_config.hf_config.model_type == "medusa":
|
||||
self.method = "medusa"
|
||||
elif self.draft_model_config.hf_config.model_type == "mlp_speculator":
|
||||
@@ -813,7 +829,18 @@ class SpeculativeConfig:
|
||||
self.draft_model_config.hf_config = eagle_config
|
||||
self.update_arch_()
|
||||
|
||||
if self.method == "dflash":
|
||||
if self.method == "dspark" and (
|
||||
"Qwen3DSparkModel" not in self.draft_model_config.architectures
|
||||
):
|
||||
# DeepSeek-V4 DSpark reuses the full DeepSeek-V4 config
|
||||
# and its weights ship in the target checkpoint.
|
||||
self.draft_model_config.hf_config.model_type = "deepseek_v4"
|
||||
self.draft_model_config.hf_config.architectures = [
|
||||
"DSparkDraftModel"
|
||||
]
|
||||
self.update_arch_()
|
||||
|
||||
if self.method in ("dflash", "dspark"):
|
||||
self.parallel_drafting = True
|
||||
|
||||
if self.num_speculative_tokens is not None and hasattr(
|
||||
@@ -1129,11 +1156,17 @@ class SpeculativeConfig:
|
||||
)
|
||||
|
||||
def use_eagle(self) -> bool:
|
||||
return self.method in ("eagle", "eagle3", "mtp", "dflash")
|
||||
# NOTE: This method is usually a stand-in for "speculative decoding using
|
||||
# target model hidden states"
|
||||
# TODO(ben): Refactor this so the naming is clearer
|
||||
return self.method in ("eagle", "eagle3", "mtp", "dflash", "dspark")
|
||||
|
||||
def use_dflash(self) -> bool:
|
||||
return self.method == "dflash"
|
||||
|
||||
def use_dspark(self) -> bool:
|
||||
return self.method == "dspark"
|
||||
|
||||
def uses_dynamic_speculative_decoding(self) -> bool:
|
||||
return self.num_speculative_tokens_per_batch_size is not None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user