[Bugfix][Model] Support tensor parallelism for DiffusionGemma (#45719) (#46177)

Signed-off-by: Carlos Alvarado <carlos-alvarado@outlook.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Carlos Alvarado Martinez
2026-06-26 20:05:04 +00:00
committed by GitHub
co-authored by Claude Lucas Wilkinson
parent dccb412e2c
commit 701a23d99f
4 changed files with 69 additions and 3 deletions
+12
View File
@@ -65,6 +65,18 @@ steps:
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt
- label: LM Eval Small Models (2xL4)
key: lm-eval-small-models-tp
timeout_in_minutes: 10
num_devices: 2
optional: true
source_file_dependencies:
- csrc/
- vllm/model_executor/layers/quantization
autorun_on_main: true
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp.txt
- label: LM Eval Large Models EP (2xB200)
key: lm-eval-large-models-ep-2xb200
timeout_in_minutes: 120
@@ -0,0 +1,10 @@
model_name: "RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic"
accuracy_threshold: 0.84
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
server_args: >-
--enforce-eager
--max-model-len 4096
--tensor-parallel-size 2
--attention-backend TRITON_ATTN
@@ -0,0 +1 @@
DiffusionGemma-26B-A4B-it-FP8-dynamic.yaml
+46 -3
View File
@@ -28,6 +28,7 @@ from transformers import AutoModel
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.distributed.parallel_state import get_tp_group
from vllm.logger import init_logger
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -499,6 +500,13 @@ def _compiled_sample_step(
ST: int,
# Sampler config
entropy_bound: float,
# Tensor-parallel vocab sharding for the self-conditioning matmul.
# ``embed_weight`` is vocab-sharded ([vocab/tp, hidden]) while ``probs``
# spans the full vocab; [sc_vocab_start, sc_vocab_end) is this rank's slice.
sc_vocab_start: int,
sc_vocab_end: int,
tp_size: int,
tp_group_name: str,
) -> torch.Tensor:
"""Compiled decode step: temperature → Gumbel sample → probs/confidence →
accept/renoise → convergence, all as vectorized PyTorch ops.
@@ -629,7 +637,17 @@ def _compiled_sample_step(
# sc_embeds directly. Storing the [.., hidden] soft embed instead of the full
# [.., vocab] probs avoids a giant persistent buffer.
sc_keep = (is_denoise & ~is_encoder_phase[decode_slots])[:, None, None]
soft_embeds = torch.matmul(probs.to(embed_weight.dtype), embed_weight) * normalizer
# Self-conditioning soft embed = probs @ embed_tokens.weight. Under tensor
# parallelism the embedding is vocab-sharded ([vocab/tp, hidden]) while
# probs spans the full vocab, so each rank multiplies its local vocab slice
# [sc_vocab_start, sc_vocab_end) and the partials are summed across ranks.
local_probs = probs[..., sc_vocab_start:sc_vocab_end].to(embed_weight.dtype)
soft_embeds = torch.matmul(
local_probs, embed_weight[: sc_vocab_end - sc_vocab_start]
)
if tp_size > 1:
soft_embeds = torch.ops.vllm.all_reduce(soft_embeds, group_name=tp_group_name)
soft_embeds = soft_embeds * normalizer
sc_embeds[decode_slots] = soft_embeds * sc_keep
# Overwrite canvas with argmax for newly converged denoise requests
@@ -843,6 +861,12 @@ class DiffusionGemmaModelState(ModelState):
raise ValueError(
f"entropy_bound must be a positive float (got {entropy_bound})"
)
# The self-conditioning matmul (probs @ embed_tokens.weight) runs over a
# vocab-parallel embedding shard. Hand the sampler this rank's vocab
# slice and TP group so it can all-reduce the partial products.
embed_tokens = self.model.model.embed_tokens
shard = embed_tokens.shard_indices
tp_group = get_tp_group()
return DiffusionSampler(
sampler=sampler,
diffusion_config=diffusion_config,
@@ -852,8 +876,12 @@ class DiffusionGemmaModelState(ModelState):
t_max=gen["t_max"],
entropy_bound=entropy_bound,
confidence_threshold=gen["confidence_threshold"],
embed_weight=self.model.model.embed_tokens.weight,
embed_weight=embed_tokens.weight,
normalizer=self.model.model.normalizer,
sc_vocab_start=shard.org_vocab_start_index,
sc_vocab_end=shard.org_vocab_end_index,
tp_size=tp_group.world_size,
tp_group_name=tp_group.unique_name,
), None
def apply_staged_writes(self) -> None:
@@ -1054,13 +1082,24 @@ class DiffusionSampler:
entropy_bound: float,
embed_weight: torch.Tensor,
normalizer: torch.Tensor,
sc_vocab_start: int = 0,
sc_vocab_end: int | None = None,
tp_size: int = 1,
tp_group_name: str = "",
):
self.sampling_states = sampler.sampling_states
self.req_states = sampler.req_states
# Self-conditioning soft embed = probs @ embed_weight * normalizer,
# computed in the sampler (see _compiled_sample_step).
# computed in the sampler (see _compiled_sample_step). ``embed_weight``
# is the vocab-parallel shard; [sc_vocab_start, sc_vocab_end) is this
# rank's slice of the full vocab and tp_* drive the cross-rank
# all-reduce.
self.embed_weight = embed_weight
self.normalizer = normalizer
self.sc_vocab_start = sc_vocab_start
self.sc_vocab_end = sc_vocab_end if sc_vocab_end is not None else vocab_size
self.tp_size = tp_size
self.tp_group_name = tp_group_name
self.canvas_length = (
diffusion_config.canvas_length if diffusion_config is not None else 32
)
@@ -1299,6 +1338,10 @@ class DiffusionSampler:
CL=self.canvas_length,
ST=states.stability_threshold,
entropy_bound=self.entropy_bound,
sc_vocab_start=self.sc_vocab_start,
sc_vocab_end=self.sc_vocab_end,
tp_size=self.tp_size,
tp_group_name=self.tp_group_name,
)
# --- Logprobs: stash on convergence, return on commit ---