diff --git a/tests/evals/gsm8k/configs/Laguna-XS-2.1-INT4.yaml b/tests/evals/gsm8k/configs/Laguna-XS-2.1-INT4.yaml new file mode 100644 index 0000000000..a6e943aa58 --- /dev/null +++ b/tests/evals/gsm8k/configs/Laguna-XS-2.1-INT4.yaml @@ -0,0 +1,11 @@ +model_name: "poolside/Laguna-XS-2.1-INT4" +accuracy_threshold: 0.90 +num_questions: 1319 +num_fewshot: 5 +max_tokens: 1024 +use_chat_completions: true +startup_max_wait_seconds: 1200 +server_args: >- + --enforce-eager + --max-model-len 4096 + --trust-remote-code diff --git a/tests/evals/gsm8k/configs/Laguna-XS.2-NVFP4.yaml b/tests/evals/gsm8k/configs/Laguna-XS.2-NVFP4.yaml index 81bf8da00b..11e6b44efd 100644 --- a/tests/evals/gsm8k/configs/Laguna-XS.2-NVFP4.yaml +++ b/tests/evals/gsm8k/configs/Laguna-XS.2-NVFP4.yaml @@ -1,7 +1,9 @@ model_name: "poolside/Laguna-XS.2-NVFP4" -accuracy_threshold: 0.86 +accuracy_threshold: 0.90 num_questions: 1319 num_fewshot: 5 +max_tokens: 1024 +use_chat_completions: true startup_max_wait_seconds: 1200 server_args: >- --enforce-eager diff --git a/tests/evals/gsm8k/configs/models-blackwell.txt b/tests/evals/gsm8k/configs/models-blackwell.txt index 5b975aa634..2e44264ef3 100644 --- a/tests/evals/gsm8k/configs/models-blackwell.txt +++ b/tests/evals/gsm8k/configs/models-blackwell.txt @@ -4,3 +4,4 @@ Qwen1.5-MoE-W4A16-CT.yaml DeepSeek-V2-Lite-Instruct-FP8.yaml Qwen3-30B-A3B-NVFP4.yaml Laguna-XS.2-NVFP4.yaml +Laguna-XS-2.1-INT4.yaml diff --git a/tests/evals/gsm8k/configs/models-small.txt b/tests/evals/gsm8k/configs/models-small.txt index ce5fe25d12..8ab287e31f 100644 --- a/tests/evals/gsm8k/configs/models-small.txt +++ b/tests/evals/gsm8k/configs/models-small.txt @@ -5,4 +5,4 @@ Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml Qwen1.5-MoE-W4A16-CT.yaml DeepSeek-V2-Lite-Instruct-FP8.yaml Qwen3-30B-A3B-MXFP4A16.yaml -gemma-4-E4B-it-qat-mobile-ct.yaml \ No newline at end of file +gemma-4-E4B-it-qat-mobile-ct.yaml