[Config] Unify indexer cache dtype under attention_config.indexer_kv_dtype (#52550)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
0ad04cff1b
commit
967e104fad
@@ -15,7 +15,7 @@ server_args: >-
|
||||
--block-size 256
|
||||
--gpu-memory-utilization 0.5
|
||||
--kv-cache-dtype fp8
|
||||
--attention_config.use_fp4_indexer_cache=True
|
||||
--attention_config.indexer_kv_dtype=mxfp4
|
||||
--max-num-batched-tokens 16384
|
||||
--max-num-seqs 128
|
||||
--speculative-config '{"method":"dspark",
|
||||
|
||||
@@ -2,4 +2,4 @@ model_name: "deepseek-ai/DeepSeek-V4-Flash"
|
||||
accuracy_threshold: 0.95
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--trust-remote-code --kv-cache-dtype fp8 --block-size 256 --enable-expert-parallel --tensor-parallel-size 2 --attention_config.use_fp4_indexer_cache=True --moe-backend deep_gemm_mega_moe --tokenizer-mode deepseek_v4 --tool-call-parser deepseek_v4 --enable-auto-tool-choice --reasoning-parser deepseek_v4 --speculative_config.method=mtp --speculative_config.num_speculative_tokens=2"
|
||||
server_args: "--trust-remote-code --kv-cache-dtype fp8 --block-size 256 --enable-expert-parallel --tensor-parallel-size 2 --attention_config.indexer_kv_dtype=mxfp4 --moe-backend deep_gemm_mega_moe --tokenizer-mode deepseek_v4 --tool-call-parser deepseek_v4 --enable-auto-tool-choice --reasoning-parser deepseek_v4 --speculative_config.method=mtp --speculative_config.num_speculative_tokens=2"
|
||||
|
||||
Reference in New Issue
Block a user