[Model] Add DiffusionGemma Support (#45163)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Martin Kukla <martin.kukla@cantab.net>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Dipika Sikka <dsikka@redhat.com>
Co-authored-by: NickLucche <nlucches@redhat.com>
Co-authored-by: jiahanc <173873397+jiahanc@users.noreply.github.com>
Co-authored-by: Alec Kohlhoff <134344302+aleckohlhoff@users.noreply.github.com>
Co-authored-by: Porras Huang <20535584+porrashuang@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: scoootscooob <167050519+scoootscooob@users.noreply.github.com>
This commit is contained in:
Lucas Wilkinson
2026-06-11 22:17:35 -07:00
committed by GitHub
co-authored by Martin Kukla Matthew Bonanni Dipika Sikka NickLucche jiahanc Alec Kohlhoff Porras Huang Claude scoootscooob
parent 1ce3cdc5c1
commit eb28452b10
52 changed files with 2695 additions and 232 deletions
+55
View File
@@ -105,6 +105,60 @@ class Gemma4Config(VerifyAndUpdateConfig):
)
class DiffusionGemmaModelForBlockDiffusionConfig(VerifyAndUpdateConfig):
@classmethod
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
"""Set up the diffusion config and defaults for DiffusionGemma.
Auto-creates DiffusionConfig from the HF config when the user
didn't pass ``--diffusion-config``. Diffusion sampling params are
read straight from generation_config.json at sampler-build time
(see DiffusionGemma's custom_sampler), not injected here.
"""
# Inherit Gemma4's attention backend selection (FA4 on Hopper,
# TRITON_ATTN fallback for heterogeneous head dims).
Gemma4Config.verify_and_update_config(vllm_config)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
attention_config = vllm_config.attention_config
if attention_config.backend == AttentionBackendEnum.FLASHINFER:
raise ValueError(
"FlashInfer does not support DiffusionGemma's mixed "
"causal/bidirectional attention. Use --attention-backend "
"FLASH_ATTN or TRITON_ATTN instead."
)
if attention_config.backend is None and not attention_config.use_non_causal:
attention_config.use_non_causal = True
logger.info(
"DiffusionGemma uses mixed causal/bidirectional attention "
"within a batch; setting use_non_causal=True to exclude "
"FlashInfer from auto-selection."
)
# Auto-create DiffusionConfig from HF config if not provided.
if vllm_config.diffusion_config is None:
from vllm.config.diffusion import DiffusionConfig
hf_config = vllm_config.model_config.hf_config
canvas_length = getattr(hf_config, "canvas_length", 256)
vllm_config.diffusion_config = DiffusionConfig(
canvas_length=canvas_length,
)
# The diffusion sampler materializes [num_seqs, canvas_length, vocab]
# fp32 transients, so concurrency is memory-bound (>8 OOMs a single H200).
# Default to 8 when the user didn't pass --max-num-seqs.
# We can't see the original None here (the engine already filled a generic
# default), so use >= DEFAULT_MAX_NUM_SEQS as a proxy, (the default is much
# larger than any deliberate value for this model)
from vllm.config.scheduler import SchedulerConfig
sc = vllm_config.scheduler_config
if sc is not None and sc.max_num_seqs >= SchedulerConfig.DEFAULT_MAX_NUM_SEQS:
sc.max_num_seqs = 8
class DeepseekV4ForCausalLMConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
@@ -591,6 +645,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"ColQwen3_5": Qwen3_5ForConditionalGenerationConfig,
"DeepseekV4ForCausalLM": DeepseekV4ForCausalLMConfig,
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
"DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501
"Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLMoeForConditionalGenerationConfig, # noqa: E501
"FalconMambaForCausalLM": MambaModelConfig,
"Gemma3TextModel": Gemma3TextModelConfig,
File diff suppressed because it is too large Load Diff
+1 -3
View File
@@ -725,10 +725,8 @@ class Gemma4DecoderLayer(nn.Module):
if self.enable_moe_block:
hidden_states_1 = self.post_feedforward_layernorm_1(hidden_states)
# Router and MoE experts see the residual (pre-MLP state),
# matching the HF transformers forward path
router_logits = self.router(residual)
hidden_states_2 = self.pre_feedforward_layernorm_2(residual)
router_logits = self.router(residual)
hidden_states_2 = self.moe(hidden_states_2, router_logits)
hidden_states_2 = self.post_feedforward_layernorm_2(hidden_states_2)
+4
View File
@@ -400,6 +400,10 @@ _MULTIMODAL_MODELS = {
"gemma3n_mm",
"Gemma3nForConditionalGeneration",
),
"DiffusionGemmaForBlockDiffusion": (
"diffusion_gemma",
"DiffusionGemmaForConditionalGeneration",
),
"Gemma4ForConditionalGeneration": ("gemma4_mm", "Gemma4ForConditionalGeneration"),
"Gemma4UnifiedForConditionalGeneration": (
"gemma4_unified",