[TurboQuant] enable FA3/FA4 for prefill paths (#40092)

Signed-off-by: 墨楼 <huangzhilin.hzl@antgroup.com>
Co-authored-by: 墨楼 <huangzhilin.hzl@antgroup.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Julian Huang
2026-04-23 07:35:24 +03:00
committed by GitHub
co-authored by 墨楼 Claude Opus 4.6 Codex
parent ccaf5ffaa3
commit fe9c3d6c5f
6 changed files with 55 additions and 15 deletions
@@ -2,4 +2,4 @@ model_name: "Qwen/Qwen3-4B"
accuracy_threshold: 0.78
num_questions: 1319
num_fewshot: 5
server_args: "--kv-cache-dtype turboquant_k3v4_nc --enforce-eager --max-model-len 4096"
server_args: "--kv-cache-dtype turboquant_k3v4_nc --max-model-len 4096"
@@ -2,4 +2,4 @@ model_name: "Qwen/Qwen3-4B"
accuracy_threshold: 0.80
num_questions: 1319
num_fewshot: 5
server_args: "--kv-cache-dtype turboquant_k8v4 --enforce-eager --max-model-len 4096"
server_args: "--kv-cache-dtype turboquant_k8v4 --max-model-len 4096"
@@ -2,4 +2,4 @@ model_name: "Qwen/Qwen3-4B"
accuracy_threshold: 0.75
num_questions: 1319
num_fewshot: 5
server_args: "--kv-cache-dtype turboquant_3bit_nc --enforce-eager --max-model-len 4096"
server_args: "--kv-cache-dtype turboquant_3bit_nc --max-model-len 4096"
@@ -2,4 +2,4 @@ model_name: "Qwen/Qwen3-4B"
accuracy_threshold: 0.80
num_questions: 1319
num_fewshot: 5
server_args: "--kv-cache-dtype turboquant_4bit_nc --enforce-eager --max-model-len 4096"
server_args: "--kv-cache-dtype turboquant_4bit_nc --max-model-len 4096"