[TurboQuant] enable FA3/FA4 for prefill paths (#40092)
Signed-off-by: 墨楼 <huangzhilin.hzl@antgroup.com> Co-authored-by: 墨楼 <huangzhilin.hzl@antgroup.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
墨楼
Claude Opus 4.6
Codex
parent
ccaf5ffaa3
commit
fe9c3d6c5f
@@ -2,4 +2,4 @@ model_name: "Qwen/Qwen3-4B"
|
||||
accuracy_threshold: 0.78
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--kv-cache-dtype turboquant_k3v4_nc --enforce-eager --max-model-len 4096"
|
||||
server_args: "--kv-cache-dtype turboquant_k3v4_nc --max-model-len 4096"
|
||||
|
||||
@@ -2,4 +2,4 @@ model_name: "Qwen/Qwen3-4B"
|
||||
accuracy_threshold: 0.80
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--kv-cache-dtype turboquant_k8v4 --enforce-eager --max-model-len 4096"
|
||||
server_args: "--kv-cache-dtype turboquant_k8v4 --max-model-len 4096"
|
||||
|
||||
@@ -2,4 +2,4 @@ model_name: "Qwen/Qwen3-4B"
|
||||
accuracy_threshold: 0.75
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--kv-cache-dtype turboquant_3bit_nc --enforce-eager --max-model-len 4096"
|
||||
server_args: "--kv-cache-dtype turboquant_3bit_nc --max-model-len 4096"
|
||||
|
||||
@@ -2,4 +2,4 @@ model_name: "Qwen/Qwen3-4B"
|
||||
accuracy_threshold: 0.80
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--kv-cache-dtype turboquant_4bit_nc --enforce-eager --max-model-len 4096"
|
||||
server_args: "--kv-cache-dtype turboquant_4bit_nc --max-model-len 4096"
|
||||
|
||||
Reference in New Issue
Block a user