[CI] Add Kimi-K3-pruned75-DSpark-TP4 gsm8k eval (#54817)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Misha Goin
2026-09-02 10:18:38 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent ba6c60e98f
commit 872084fb77
5 changed files with 64 additions and 30 deletions
+3 -1
View File
@@ -109,9 +109,11 @@ steps:
num_devices: 4
optional: true
source_file_dependencies:
- tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml
- tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TEP4.yaml
- tests/evals/gsm8k/configs/Kimi-K3-pruned75-DSpark-TP4.yaml
- tests/evals/gsm8k/configs/models-spec-decode.txt
- vllm/models/deepseek_v4/
- vllm/models/kimi_k3/
- vllm/model_executor/models/qwen3_dspark.py
- vllm/v1/spec_decode/metrics.py
- vllm/v1/worker/gpu/spec_decode/adaptive_verification.py
@@ -0,0 +1,26 @@
model_name: "mgoin/Kimi-K3-pruned75"
accuracy_threshold: 0.33
min_acceptance_length: 4.8
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1800
server_args: >-
--trust-remote-code
--gpu-memory-utilization 0.90
--tensor-parallel-size 4
--load-format fastsafetensors
--no-enable-flashinfer-autotune
--max-model-len 1048576
--max-num-batched-tokens 16384
--max-num-seqs 128
--kv-cache-dtype fp8
--attention-config '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}'
--enable-prefix-caching
--enable-auto-tool-choice
--tool-call-parser kimi_k3
--reasoning-parser kimi_k3
--speculative-config '{"method":"dspark",
"model":"RedHatAI/Kimi-K3-speculator.dspark",
"attention_backend":"FLASH_ATTN","num_speculative_tokens":8,
"draft_sample_method":"probabilistic",
"rejection_sample_method":"block"}'
@@ -1 +1,2 @@
DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml
DeepSeek-V4-Flash-DSpark-confidence-TEP4.yaml
Kimi-K3-pruned75-DSpark-TP4.yaml
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, ClassVar
import torch
import vllm.envs as envs
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.v1.attention.backends.mla.prefill.base import (
MLADimensions,
MLAPrefillBackend,
@@ -161,39 +162,43 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
kv_indptr = qo_indptr.clone()
assert self._prefill_main is not None
self._prefill_main.plan(
qo_indptr=qo_indptr,
kv_indptr=kv_indptr,
num_qo_heads=num_qo_heads,
num_kv_heads=num_kv_heads,
head_dim_qk=head_dim_qk,
head_dim_vo=head_dim_vo,
causal=True,
sm_scale=global_hyperparameters.sm_scale,
window_left=global_hyperparameters.window_left,
logits_soft_cap=global_hyperparameters.logits_soft_cap,
q_data_type=prefill_metadata.q_data_type,
o_data_type=prefill_metadata.output_dtype,
)
# FlashInfer's plan() currently copies GPU indptrs to the CPU.
# This will be fixed by https://github.com/vllm-project/vllm/pull/52657.
with gpu_sync_allowed():
self._prefill_main.plan(
qo_indptr=qo_indptr,
kv_indptr=kv_indptr,
num_qo_heads=num_qo_heads,
num_kv_heads=num_kv_heads,
head_dim_qk=head_dim_qk,
head_dim_vo=head_dim_vo,
causal=True,
sm_scale=global_hyperparameters.sm_scale,
window_left=global_hyperparameters.window_left,
logits_soft_cap=global_hyperparameters.logits_soft_cap,
q_data_type=prefill_metadata.q_data_type,
o_data_type=prefill_metadata.output_dtype,
)
if has_context:
chunked_context = prefill_metadata.chunked_context
assert chunked_context is not None
for chunk in chunked_context.chunks:
self._prefill_chunks[chunk.index].plan(
qo_indptr=chunk.query_start_loc,
kv_indptr=chunk.cu_seq_lens,
num_qo_heads=num_qo_heads,
num_kv_heads=num_kv_heads,
head_dim_qk=head_dim_qk,
head_dim_vo=head_dim_vo,
causal=False,
sm_scale=global_hyperparameters.sm_scale,
window_left=global_hyperparameters.window_left,
logits_soft_cap=global_hyperparameters.logits_soft_cap,
q_data_type=prefill_metadata.q_data_type,
o_data_type=prefill_metadata.output_dtype,
)
with gpu_sync_allowed():
self._prefill_chunks[chunk.index].plan(
qo_indptr=chunk.query_start_loc,
kv_indptr=chunk.cu_seq_lens,
num_qo_heads=num_qo_heads,
num_kv_heads=num_kv_heads,
head_dim_qk=head_dim_qk,
head_dim_vo=head_dim_vo,
causal=False,
sm_scale=global_hyperparameters.sm_scale,
window_left=global_hyperparameters.window_left,
logits_soft_cap=global_hyperparameters.logits_soft_cap,
q_data_type=prefill_metadata.q_data_type,
o_data_type=prefill_metadata.output_dtype,
)
def supports_out(self) -> bool:
# Planned with head_dim_vo == v_head_dim, so the output is unpadded.