diff --git a/tests/evals/gsm8k/configs/DiffusionGemma-26B-A4B-it-FP8-dynamic.yaml b/tests/evals/gsm8k/configs/DiffusionGemma-26B-A4B-it-FP8-dynamic.yaml index 8d412b7c4b..060c850681 100644 --- a/tests/evals/gsm8k/configs/DiffusionGemma-26B-A4B-it-FP8-dynamic.yaml +++ b/tests/evals/gsm8k/configs/DiffusionGemma-26B-A4B-it-FP8-dynamic.yaml @@ -4,6 +4,10 @@ num_questions: 1319 num_fewshot: 5 startup_max_wait_seconds: 1200 use_chat_completions: true +# Diffusion models use a fixed internal temperature schedule and do not +# support per-request temperature or seed overrides. +temperature: 1.0 +seed: null server_args: >- --enforce-eager --max-model-len 4096 diff --git a/tests/evals/gsm8k/test_gsm8k_correctness.py b/tests/evals/gsm8k/test_gsm8k_correctness.py index 0c48af6d3c..0a7fd254ee 100644 --- a/tests/evals/gsm8k/test_gsm8k_correctness.py +++ b/tests/evals/gsm8k/test_gsm8k_correctness.py @@ -69,6 +69,8 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict: use_chat_completions=eval_config.get("use_chat_completions", False), host=host, port=port, + temperature=eval_config.get("temperature", 0.0), + seed=eval_config.get("seed", 42), request_timeout_seconds=request_timeout_seconds, gen_prefix=eval_config.get("gen_prefix", ""), )