[PERF] [Qwen3.5] Split mixed prefill+decode batches: route decodes to the recurrent kernel (#44700)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
67d3792d99
commit
fa27d4e9cf
@@ -0,0 +1,12 @@
|
||||
model_name: "nvidia/Qwen3.5-397B-A17B-NVFP4"
|
||||
accuracy_threshold: 0.88
|
||||
tolerance: 0.03
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--max-model-len 4096
|
||||
--data-parallel-size 2
|
||||
--enable-expert-parallel
|
||||
--max-num-seqs 384
|
||||
--spec-method mtp
|
||||
--spec-tokens 3
|
||||
@@ -1,3 +1,4 @@
|
||||
Qwen3.5-35B-A3B-DEP2.yaml
|
||||
Qwen3.5-35B-A3B-FP8-DEP2.yaml
|
||||
Qwen3.5-397B-A17B-NVFP4-DEP2.yaml
|
||||
Qwen3.5-397B-A17B-NVFP4-DEP2.yaml
|
||||
Qwen3.5-397B-A17B-NVFP4-DEP2-MTP.yaml
|
||||
Reference in New Issue
Block a user