[CI] Add Kimi-K3-pruned75-DSpark-TP4 gsm8k eval (#54817)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
ba6c60e98f
commit
872084fb77
@@ -109,9 +109,11 @@ steps:
|
||||
num_devices: 4
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
- tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml
|
||||
- tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TEP4.yaml
|
||||
- tests/evals/gsm8k/configs/Kimi-K3-pruned75-DSpark-TP4.yaml
|
||||
- tests/evals/gsm8k/configs/models-spec-decode.txt
|
||||
- vllm/models/deepseek_v4/
|
||||
- vllm/models/kimi_k3/
|
||||
- vllm/model_executor/models/qwen3_dspark.py
|
||||
- vllm/v1/spec_decode/metrics.py
|
||||
- vllm/v1/worker/gpu/spec_decode/adaptive_verification.py
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
model_name: "mgoin/Kimi-K3-pruned75"
|
||||
accuracy_threshold: 0.33
|
||||
min_acceptance_length: 4.8
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
startup_max_wait_seconds: 1800
|
||||
server_args: >-
|
||||
--trust-remote-code
|
||||
--gpu-memory-utilization 0.90
|
||||
--tensor-parallel-size 4
|
||||
--load-format fastsafetensors
|
||||
--no-enable-flashinfer-autotune
|
||||
--max-model-len 1048576
|
||||
--max-num-batched-tokens 16384
|
||||
--max-num-seqs 128
|
||||
--kv-cache-dtype fp8
|
||||
--attention-config '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}'
|
||||
--enable-prefix-caching
|
||||
--enable-auto-tool-choice
|
||||
--tool-call-parser kimi_k3
|
||||
--reasoning-parser kimi_k3
|
||||
--speculative-config '{"method":"dspark",
|
||||
"model":"RedHatAI/Kimi-K3-speculator.dspark",
|
||||
"attention_backend":"FLASH_ATTN","num_speculative_tokens":8,
|
||||
"draft_sample_method":"probabilistic",
|
||||
"rejection_sample_method":"block"}'
|
||||
@@ -1 +1,2 @@
|
||||
DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml
|
||||
DeepSeek-V4-Flash-DSpark-confidence-TEP4.yaml
|
||||
Kimi-K3-pruned75-DSpark-TP4.yaml
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, ClassVar
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.v1.attention.backends.mla.prefill.base import (
|
||||
MLADimensions,
|
||||
MLAPrefillBackend,
|
||||
@@ -161,39 +162,43 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
|
||||
kv_indptr = qo_indptr.clone()
|
||||
|
||||
assert self._prefill_main is not None
|
||||
self._prefill_main.plan(
|
||||
qo_indptr=qo_indptr,
|
||||
kv_indptr=kv_indptr,
|
||||
num_qo_heads=num_qo_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim_qk=head_dim_qk,
|
||||
head_dim_vo=head_dim_vo,
|
||||
causal=True,
|
||||
sm_scale=global_hyperparameters.sm_scale,
|
||||
window_left=global_hyperparameters.window_left,
|
||||
logits_soft_cap=global_hyperparameters.logits_soft_cap,
|
||||
q_data_type=prefill_metadata.q_data_type,
|
||||
o_data_type=prefill_metadata.output_dtype,
|
||||
)
|
||||
# FlashInfer's plan() currently copies GPU indptrs to the CPU.
|
||||
# This will be fixed by https://github.com/vllm-project/vllm/pull/52657.
|
||||
with gpu_sync_allowed():
|
||||
self._prefill_main.plan(
|
||||
qo_indptr=qo_indptr,
|
||||
kv_indptr=kv_indptr,
|
||||
num_qo_heads=num_qo_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim_qk=head_dim_qk,
|
||||
head_dim_vo=head_dim_vo,
|
||||
causal=True,
|
||||
sm_scale=global_hyperparameters.sm_scale,
|
||||
window_left=global_hyperparameters.window_left,
|
||||
logits_soft_cap=global_hyperparameters.logits_soft_cap,
|
||||
q_data_type=prefill_metadata.q_data_type,
|
||||
o_data_type=prefill_metadata.output_dtype,
|
||||
)
|
||||
|
||||
if has_context:
|
||||
chunked_context = prefill_metadata.chunked_context
|
||||
assert chunked_context is not None
|
||||
for chunk in chunked_context.chunks:
|
||||
self._prefill_chunks[chunk.index].plan(
|
||||
qo_indptr=chunk.query_start_loc,
|
||||
kv_indptr=chunk.cu_seq_lens,
|
||||
num_qo_heads=num_qo_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim_qk=head_dim_qk,
|
||||
head_dim_vo=head_dim_vo,
|
||||
causal=False,
|
||||
sm_scale=global_hyperparameters.sm_scale,
|
||||
window_left=global_hyperparameters.window_left,
|
||||
logits_soft_cap=global_hyperparameters.logits_soft_cap,
|
||||
q_data_type=prefill_metadata.q_data_type,
|
||||
o_data_type=prefill_metadata.output_dtype,
|
||||
)
|
||||
with gpu_sync_allowed():
|
||||
self._prefill_chunks[chunk.index].plan(
|
||||
qo_indptr=chunk.query_start_loc,
|
||||
kv_indptr=chunk.cu_seq_lens,
|
||||
num_qo_heads=num_qo_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim_qk=head_dim_qk,
|
||||
head_dim_vo=head_dim_vo,
|
||||
causal=False,
|
||||
sm_scale=global_hyperparameters.sm_scale,
|
||||
window_left=global_hyperparameters.window_left,
|
||||
logits_soft_cap=global_hyperparameters.logits_soft_cap,
|
||||
q_data_type=prefill_metadata.q_data_type,
|
||||
o_data_type=prefill_metadata.output_dtype,
|
||||
)
|
||||
|
||||
def supports_out(self) -> bool:
|
||||
# Planned with head_dim_vo == v_head_dim, so the output is unpadded.
|
||||
|
||||
Reference in New Issue
Block a user