[Spec Decode] DSpark (#46995)

Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Benjamin Chislett
2026-07-01 08:30:24 -07:00
committed by GitHub
co-authored by Giancarlo Delfin mgoin
parent 63fcce4de1
commit f5a8d73377
24 changed files with 1821 additions and 94 deletions
+36 -3
View File
@@ -54,6 +54,7 @@ MTPModelTypes = Literal[
]
NgramGPUTypes = Literal["ngram_gpu"]
DFlashModelTypes = Literal["dflash"]
DSparkModelTypes = Literal["dspark"]
EagleModelTypes = Literal[
"eagle", "eagle3", "extract_hidden_states", MTPModelTypes, DFlashModelTypes
]
@@ -66,6 +67,7 @@ SpeculativeMethod = Literal[
"custom_class",
EagleModelTypes,
NgramGPUTypes,
DSparkModelTypes,
]
RejectionSampleMethod = Literal["standard", "synthetic", "block"]
DraftSampleMethod = Literal["greedy", "probabilistic"]
@@ -291,6 +293,7 @@ class SpeculativeConfig:
"eagle3",
"extract_hidden_states",
"dflash",
"dspark",
)
factors.append(uses_aux_hidden_states)
@@ -608,6 +611,13 @@ class SpeculativeConfig:
# --quantization fp8 with a bf16 checkpoint.
if not self.quantization:
self.quantization = self.target_model_config.quantization
elif self.method == "dspark":
# DeepSeek DSpark can ship the weights inside the target checkpoint
if self.target_model_config is None:
raise ValueError("target_model_config must be present for dspark")
self.model = self.target_model_config.model
if not self.quantization:
self.quantization = self.target_model_config.quantization
elif self.method in ("ngram", "[ngram]"):
self.model = "ngram"
elif self.method == "ngram_gpu":
@@ -755,18 +765,24 @@ class SpeculativeConfig:
draft_hf.truncated_vocab_size = target_vocab
# Automatically detect the method
if self.method in ("eagle", "eagle3", "dflash"):
if self.method in ("eagle", "eagle3", "dflash", "dspark"):
pass
# examples:
# yuhuili/EAGLE-LLaMA3-Instruct-8B
# yuhuili/EAGLE3-LLaMA3.1-Instruct-8B
# AngelSlim/Qwen3-8B_eagle3
# deepseek-ai/dspark_qwen3_8b_block7
elif "eagle-" in self.draft_model_config.model.lower():
self.method = "eagle"
elif "eagle3" in self.draft_model_config.model.lower():
self.method = "eagle3"
elif "dflash" in self.draft_model_config.model.lower():
self.method = "dflash"
elif (
"dspark" in self.draft_model_config.model.lower()
or "Qwen3DSparkModel" in self.draft_model_config.architectures
):
self.method = "dspark"
elif self.draft_model_config.hf_config.model_type == "medusa":
self.method = "medusa"
elif self.draft_model_config.hf_config.model_type == "mlp_speculator":
@@ -813,7 +829,18 @@ class SpeculativeConfig:
self.draft_model_config.hf_config = eagle_config
self.update_arch_()
if self.method == "dflash":
if self.method == "dspark" and (
"Qwen3DSparkModel" not in self.draft_model_config.architectures
):
# DeepSeek-V4 DSpark reuses the full DeepSeek-V4 config
# and its weights ship in the target checkpoint.
self.draft_model_config.hf_config.model_type = "deepseek_v4"
self.draft_model_config.hf_config.architectures = [
"DSparkDraftModel"
]
self.update_arch_()
if self.method in ("dflash", "dspark"):
self.parallel_drafting = True
if self.num_speculative_tokens is not None and hasattr(
@@ -1129,11 +1156,17 @@ class SpeculativeConfig:
)
def use_eagle(self) -> bool:
return self.method in ("eagle", "eagle3", "mtp", "dflash")
# NOTE: This method is usually a stand-in for "speculative decoding using
# target model hidden states"
# TODO(ben): Refactor this so the naming is clearer
return self.method in ("eagle", "eagle3", "mtp", "dflash", "dspark")
def use_dflash(self) -> bool:
return self.method == "dflash"
def use_dspark(self) -> bool:
return self.method == "dspark"
def uses_dynamic_speculative_decoding(self) -> bool:
return self.num_speculative_tokens_per_batch_size is not None