[Model] Add DiffusionGemma Support (#45163)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Martin Kukla <martin.kukla@cantab.net> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Dipika Sikka <dsikka@redhat.com> Co-authored-by: NickLucche <nlucches@redhat.com> Co-authored-by: jiahanc <173873397+jiahanc@users.noreply.github.com> Co-authored-by: Alec Kohlhoff <134344302+aleckohlhoff@users.noreply.github.com> Co-authored-by: Porras Huang <20535584+porrashuang@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: scoootscooob <167050519+scoootscooob@users.noreply.github.com>
This commit is contained in:
co-authored by
Martin Kukla
Matthew Bonanni
Dipika Sikka
NickLucche
jiahanc
Alec Kohlhoff
Porras Huang
Claude
scoootscooob
parent
1ce3cdc5c1
commit
eb28452b10
@@ -188,6 +188,7 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
]
|
||||
@@ -267,6 +268,7 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
activation_str_to_value_map = {
|
||||
MoEActivation.SILU: ActivationType.Swiglu, # This is the default
|
||||
MoEActivation.GELU_TANH: ActivationType.Geglu,
|
||||
MoEActivation.SWIGLUOAI: ActivationType.Swiglu, # gpt-oss alias
|
||||
MoEActivation.RELU2_NO_MUL: ActivationType.Relu2,
|
||||
}
|
||||
|
||||
@@ -142,6 +142,7 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -34,6 +34,7 @@ def activation_to_flashinfer_type(activation: MoEActivation) -> "ActivationType"
|
||||
MoEActivation.GELU_NO_MUL: ActivationType.Gelu,
|
||||
MoEActivation.SILU: ActivationType.Swiglu,
|
||||
MoEActivation.GELU: ActivationType.Geglu,
|
||||
MoEActivation.GELU_TANH: ActivationType.Geglu,
|
||||
MoEActivation.RELU2_NO_MUL: ActivationType.Relu2,
|
||||
}
|
||||
return ACTIVATION_TO_FI_ACTIVATION[activation]
|
||||
|
||||
@@ -105,6 +105,60 @@ class Gemma4Config(VerifyAndUpdateConfig):
|
||||
)
|
||||
|
||||
|
||||
class DiffusionGemmaModelForBlockDiffusionConfig(VerifyAndUpdateConfig):
|
||||
@classmethod
|
||||
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
|
||||
"""Set up the diffusion config and defaults for DiffusionGemma.
|
||||
|
||||
Auto-creates DiffusionConfig from the HF config when the user
|
||||
didn't pass ``--diffusion-config``. Diffusion sampling params are
|
||||
read straight from generation_config.json at sampler-build time
|
||||
(see DiffusionGemma's custom_sampler), not injected here.
|
||||
"""
|
||||
# Inherit Gemma4's attention backend selection (FA4 on Hopper,
|
||||
# TRITON_ATTN fallback for heterogeneous head dims).
|
||||
Gemma4Config.verify_and_update_config(vllm_config)
|
||||
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
attention_config = vllm_config.attention_config
|
||||
if attention_config.backend == AttentionBackendEnum.FLASHINFER:
|
||||
raise ValueError(
|
||||
"FlashInfer does not support DiffusionGemma's mixed "
|
||||
"causal/bidirectional attention. Use --attention-backend "
|
||||
"FLASH_ATTN or TRITON_ATTN instead."
|
||||
)
|
||||
if attention_config.backend is None and not attention_config.use_non_causal:
|
||||
attention_config.use_non_causal = True
|
||||
logger.info(
|
||||
"DiffusionGemma uses mixed causal/bidirectional attention "
|
||||
"within a batch; setting use_non_causal=True to exclude "
|
||||
"FlashInfer from auto-selection."
|
||||
)
|
||||
|
||||
# Auto-create DiffusionConfig from HF config if not provided.
|
||||
if vllm_config.diffusion_config is None:
|
||||
from vllm.config.diffusion import DiffusionConfig
|
||||
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
canvas_length = getattr(hf_config, "canvas_length", 256)
|
||||
vllm_config.diffusion_config = DiffusionConfig(
|
||||
canvas_length=canvas_length,
|
||||
)
|
||||
|
||||
# The diffusion sampler materializes [num_seqs, canvas_length, vocab]
|
||||
# fp32 transients, so concurrency is memory-bound (>8 OOMs a single H200).
|
||||
# Default to 8 when the user didn't pass --max-num-seqs.
|
||||
# We can't see the original None here (the engine already filled a generic
|
||||
# default), so use >= DEFAULT_MAX_NUM_SEQS as a proxy, (the default is much
|
||||
# larger than any deliberate value for this model)
|
||||
from vllm.config.scheduler import SchedulerConfig
|
||||
|
||||
sc = vllm_config.scheduler_config
|
||||
if sc is not None and sc.max_num_seqs >= SchedulerConfig.DEFAULT_MAX_NUM_SEQS:
|
||||
sc.max_num_seqs = 8
|
||||
|
||||
|
||||
class DeepseekV4ForCausalLMConfig(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
@@ -591,6 +645,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"ColQwen3_5": Qwen3_5ForConditionalGenerationConfig,
|
||||
"DeepseekV4ForCausalLM": DeepseekV4ForCausalLMConfig,
|
||||
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
|
||||
"DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501
|
||||
"Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLMoeForConditionalGenerationConfig, # noqa: E501
|
||||
"FalconMambaForCausalLM": MambaModelConfig,
|
||||
"Gemma3TextModel": Gemma3TextModelConfig,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -725,10 +725,8 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
if self.enable_moe_block:
|
||||
hidden_states_1 = self.post_feedforward_layernorm_1(hidden_states)
|
||||
|
||||
# Router and MoE experts see the residual (pre-MLP state),
|
||||
# matching the HF transformers forward path
|
||||
router_logits = self.router(residual)
|
||||
hidden_states_2 = self.pre_feedforward_layernorm_2(residual)
|
||||
router_logits = self.router(residual)
|
||||
hidden_states_2 = self.moe(hidden_states_2, router_logits)
|
||||
hidden_states_2 = self.post_feedforward_layernorm_2(hidden_states_2)
|
||||
|
||||
|
||||
@@ -400,6 +400,10 @@ _MULTIMODAL_MODELS = {
|
||||
"gemma3n_mm",
|
||||
"Gemma3nForConditionalGeneration",
|
||||
),
|
||||
"DiffusionGemmaForBlockDiffusion": (
|
||||
"diffusion_gemma",
|
||||
"DiffusionGemmaForConditionalGeneration",
|
||||
),
|
||||
"Gemma4ForConditionalGeneration": ("gemma4_mm", "Gemma4ForConditionalGeneration"),
|
||||
"Gemma4UnifiedForConditionalGeneration": (
|
||||
"gemma4_unified",
|
||||
|
||||
Reference in New Issue
Block a user