[PCP][DCP] Enable PCP+DCP on sparse-MLA models (#56157)
Signed-off-by: patryk.saffer <patryk.saffer@blc-login-1.blc1.mistralcompute.ai> Signed-off-by: patryk.saffer <patryk.saffer@blc-login-0.blc1.mistralcompute.ai> Signed-off-by: PatrykSaffer <patryk.saffer@mistral.ai> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: patryk.saffer <patryk.saffer@blc-login-1.blc1.mistralcompute.ai> Co-authored-by: patryk.saffer <patryk.saffer@blc-login-0.blc1.mistralcompute.ai> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Patryk Saffer <PatrykSaffer@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
patryk.saffer
patryk.saffer
Lucas Wilkinson
Patryk Saffer
Claude Fable 5.1
parent
72d4d83157
commit
ebe1dec2da
@@ -0,0 +1,19 @@
|
||||
model_name: "nvidia/GLM-5.2-NVFP4"
|
||||
accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
max_concurrency: 100
|
||||
server_args: >-
|
||||
--compilation-config '{"cudagraph_mode":"PIECEWISE"}'
|
||||
--max-model-len 4096
|
||||
--max-num-batched-tokens 32768
|
||||
--safetensors-load-strategy prefetch
|
||||
--moe-backend flashinfer_cutlass
|
||||
--prefill-context-parallel-size 4
|
||||
--decode-context-parallel-size 4
|
||||
--enable-expert-parallel
|
||||
--kv-cache-dtype fp8
|
||||
env:
|
||||
PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True"
|
||||
VLLM_LOGGING_LEVEL: "DEBUG"
|
||||
VLLM_USE_V2_MODEL_RUNNER: "1"
|
||||
@@ -1,2 +1,3 @@
|
||||
GLM-5.2-NVFP4-TP2-PCP2-EP.yaml
|
||||
GLM-5.2-NVFP4-TP1-PCP4-EP.yaml
|
||||
GLM-5.2-NVFP4-TP1-PCP4-DCP4-EP.yaml
|
||||
|
||||
Reference in New Issue
Block a user