[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity (#38479)
Signed-off-by: vibhavagarwal5 <vibhavagarwal5@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Xinyu Chen <xinyu1.chen@intel.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Xinyu Chen
Michael Goin
parent
3bfe55a037
commit
f4b42df048
@@ -0,0 +1,5 @@
|
||||
model_name: "Qwen/Qwen3-4B"
|
||||
accuracy_threshold: 0.78
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--kv-cache-dtype turboquant_k3v4_nc --enforce-eager --max-model-len 4096"
|
||||
@@ -0,0 +1,5 @@
|
||||
model_name: "Qwen/Qwen3-4B"
|
||||
accuracy_threshold: 0.80
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--kv-cache-dtype turboquant_k8v4 --enforce-eager --max-model-len 4096"
|
||||
@@ -0,0 +1,5 @@
|
||||
model_name: "Qwen/Qwen3-4B"
|
||||
accuracy_threshold: 0.75
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--kv-cache-dtype turboquant_3bit_nc --enforce-eager --max-model-len 4096"
|
||||
@@ -0,0 +1,5 @@
|
||||
model_name: "Qwen/Qwen3-4B"
|
||||
accuracy_threshold: 0.80
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--kv-cache-dtype turboquant_4bit_nc --enforce-eager --max-model-len 4096"
|
||||
@@ -0,0 +1,4 @@
|
||||
Qwen3-4B-TQ-k8v4.yaml
|
||||
Qwen3-4B-TQ-t4nc.yaml
|
||||
Qwen3-4B-TQ-k3v4nc.yaml
|
||||
Qwen3-4B-TQ-t3nc.yaml
|
||||
Reference in New Issue
Block a user