[Spec Decode] Enable Gemma4 DSpark adaptive verification with FlashInfer (#57263)

Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
qizixi
2026-09-26 17:01:17 +08:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5.5 Nick Hill
parent 3b4566c5cf
commit a4eb3f25d6
24 changed files with 894 additions and 135 deletions
+22 -3
View File
@@ -24,7 +24,11 @@ from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
from .gemma4_mtp import Gemma4MTPAttention, Gemma4MTPDecoderLayer
from .qwen3_dflash import DFlashQwen3Model, _dflash_layer_causal
from .qwen3_dspark import DSparkMarkovHead, Qwen3DSparkForCausalLM
from .qwen3_dspark import (
DSparkConfidenceHead,
DSparkMarkovHead,
Qwen3DSparkForCausalLM,
)
from .utils import extract_layer_index, maybe_prefix
@@ -187,6 +191,18 @@ class Gemma4DSparkModel(DFlashQwen3Model):
config.markov_rank,
prefix=maybe_prefix(prefix, "markov_head"),
)
self.confidence_head: DSparkConfidenceHead | None = None
if getattr(config, "enable_confidence_head", False):
with_markov = getattr(config, "confidence_head_with_markov", False)
input_dim = config.hidden_size
if with_markov:
input_dim += config.markov_rank
self.confidence_head = DSparkConfidenceHead(
input_dim,
prefix=maybe_prefix(prefix, "confidence_head"),
bias=True,
with_markov=with_markov,
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids) * self.normalizer
@@ -295,9 +311,8 @@ class Gemma4DSparkForCausalLM(Qwen3DSparkForCausalLM):
params = dict(self.named_parameters())
params.update(dict(self.named_buffers()))
loaded: set[str] = set()
includes_confidence_head = False
for name, w in weights:
if "confidence_head" in name:
continue
if "lm_head" not in name:
name = "model." + name
for pn, wn, shard in stacked:
@@ -310,5 +325,9 @@ class Gemma4DSparkForCausalLM(Qwen3DSparkForCausalLM):
p = params[name]
getattr(p, "weight_loader", default_weight_loader)(p, w)
loaded.add(name)
if "confidence_head" in name:
includes_confidence_head = True
if not includes_confidence_head:
self.model.confidence_head = None
self.model._build_fused_kv_buffers()
return loaded