diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 8063d5e72f..793b9d8913 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -65,6 +65,18 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt +- label: LM Eval Small Models (2xL4) + key: lm-eval-small-models-tp + timeout_in_minutes: 10 + num_devices: 2 + optional: true + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + autorun_on_main: true + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp.txt + - label: LM Eval Large Models EP (2xB200) key: lm-eval-large-models-ep-2xb200 timeout_in_minutes: 120 diff --git a/tests/evals/gsm8k/configs/DiffusionGemma-26B-A4B-it-FP8-dynamic.yaml b/tests/evals/gsm8k/configs/DiffusionGemma-26B-A4B-it-FP8-dynamic.yaml new file mode 100644 index 0000000000..3304cbff65 --- /dev/null +++ b/tests/evals/gsm8k/configs/DiffusionGemma-26B-A4B-it-FP8-dynamic.yaml @@ -0,0 +1,10 @@ +model_name: "RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic" +accuracy_threshold: 0.84 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +server_args: >- + --enforce-eager + --max-model-len 4096 + --tensor-parallel-size 2 + --attention-backend TRITON_ATTN diff --git a/tests/evals/gsm8k/configs/models-small-tp.txt b/tests/evals/gsm8k/configs/models-small-tp.txt new file mode 100644 index 0000000000..63bba5bcd1 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-small-tp.txt @@ -0,0 +1 @@ +DiffusionGemma-26B-A4B-it-FP8-dynamic.yaml diff --git a/vllm/model_executor/models/diffusion_gemma.py b/vllm/model_executor/models/diffusion_gemma.py index 85e0ef0467..5c4dd8eb55 100644 --- a/vllm/model_executor/models/diffusion_gemma.py +++ b/vllm/model_executor/models/diffusion_gemma.py @@ -28,6 +28,7 @@ from transformers import AutoModel from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode +from vllm.distributed.parallel_state import get_tp_group from vllm.logger import init_logger from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -499,6 +500,13 @@ def _compiled_sample_step( ST: int, # Sampler config entropy_bound: float, + # Tensor-parallel vocab sharding for the self-conditioning matmul. + # ``embed_weight`` is vocab-sharded ([vocab/tp, hidden]) while ``probs`` + # spans the full vocab; [sc_vocab_start, sc_vocab_end) is this rank's slice. + sc_vocab_start: int, + sc_vocab_end: int, + tp_size: int, + tp_group_name: str, ) -> torch.Tensor: """Compiled decode step: temperature → Gumbel sample → probs/confidence → accept/renoise → convergence, all as vectorized PyTorch ops. @@ -629,7 +637,17 @@ def _compiled_sample_step( # sc_embeds directly. Storing the [.., hidden] soft embed instead of the full # [.., vocab] probs avoids a giant persistent buffer. sc_keep = (is_denoise & ~is_encoder_phase[decode_slots])[:, None, None] - soft_embeds = torch.matmul(probs.to(embed_weight.dtype), embed_weight) * normalizer + # Self-conditioning soft embed = probs @ embed_tokens.weight. Under tensor + # parallelism the embedding is vocab-sharded ([vocab/tp, hidden]) while + # probs spans the full vocab, so each rank multiplies its local vocab slice + # [sc_vocab_start, sc_vocab_end) and the partials are summed across ranks. + local_probs = probs[..., sc_vocab_start:sc_vocab_end].to(embed_weight.dtype) + soft_embeds = torch.matmul( + local_probs, embed_weight[: sc_vocab_end - sc_vocab_start] + ) + if tp_size > 1: + soft_embeds = torch.ops.vllm.all_reduce(soft_embeds, group_name=tp_group_name) + soft_embeds = soft_embeds * normalizer sc_embeds[decode_slots] = soft_embeds * sc_keep # Overwrite canvas with argmax for newly converged denoise requests @@ -843,6 +861,12 @@ class DiffusionGemmaModelState(ModelState): raise ValueError( f"entropy_bound must be a positive float (got {entropy_bound})" ) + # The self-conditioning matmul (probs @ embed_tokens.weight) runs over a + # vocab-parallel embedding shard. Hand the sampler this rank's vocab + # slice and TP group so it can all-reduce the partial products. + embed_tokens = self.model.model.embed_tokens + shard = embed_tokens.shard_indices + tp_group = get_tp_group() return DiffusionSampler( sampler=sampler, diffusion_config=diffusion_config, @@ -852,8 +876,12 @@ class DiffusionGemmaModelState(ModelState): t_max=gen["t_max"], entropy_bound=entropy_bound, confidence_threshold=gen["confidence_threshold"], - embed_weight=self.model.model.embed_tokens.weight, + embed_weight=embed_tokens.weight, normalizer=self.model.model.normalizer, + sc_vocab_start=shard.org_vocab_start_index, + sc_vocab_end=shard.org_vocab_end_index, + tp_size=tp_group.world_size, + tp_group_name=tp_group.unique_name, ), None def apply_staged_writes(self) -> None: @@ -1054,13 +1082,24 @@ class DiffusionSampler: entropy_bound: float, embed_weight: torch.Tensor, normalizer: torch.Tensor, + sc_vocab_start: int = 0, + sc_vocab_end: int | None = None, + tp_size: int = 1, + tp_group_name: str = "", ): self.sampling_states = sampler.sampling_states self.req_states = sampler.req_states # Self-conditioning soft embed = probs @ embed_weight * normalizer, - # computed in the sampler (see _compiled_sample_step). + # computed in the sampler (see _compiled_sample_step). ``embed_weight`` + # is the vocab-parallel shard; [sc_vocab_start, sc_vocab_end) is this + # rank's slice of the full vocab and tp_* drive the cross-rank + # all-reduce. self.embed_weight = embed_weight self.normalizer = normalizer + self.sc_vocab_start = sc_vocab_start + self.sc_vocab_end = sc_vocab_end if sc_vocab_end is not None else vocab_size + self.tp_size = tp_size + self.tp_group_name = tp_group_name self.canvas_length = ( diffusion_config.canvas_length if diffusion_config is not None else 32 ) @@ -1299,6 +1338,10 @@ class DiffusionSampler: CL=self.canvas_length, ST=states.stability_threshold, entropy_bound=self.entropy_bound, + sc_vocab_start=self.sc_vocab_start, + sc_vocab_end=self.sc_vocab_end, + tp_size=self.tp_size, + tp_group_name=self.tp_group_name, ) # --- Logprobs: stash on convergence, return on commit ---