diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 1b43c707eb..00d99ff7fd 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -109,9 +109,11 @@ steps: num_devices: 4 optional: true source_file_dependencies: - - tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml + - tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TEP4.yaml + - tests/evals/gsm8k/configs/Kimi-K3-pruned75-DSpark-TP4.yaml - tests/evals/gsm8k/configs/models-spec-decode.txt - vllm/models/deepseek_v4/ + - vllm/models/kimi_k3/ - vllm/model_executor/models/qwen3_dspark.py - vllm/v1/spec_decode/metrics.py - vllm/v1/worker/gpu/spec_decode/adaptive_verification.py diff --git a/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml b/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TEP4.yaml similarity index 100% rename from tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml rename to tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TEP4.yaml diff --git a/tests/evals/gsm8k/configs/Kimi-K3-pruned75-DSpark-TP4.yaml b/tests/evals/gsm8k/configs/Kimi-K3-pruned75-DSpark-TP4.yaml new file mode 100644 index 0000000000..d92bafd62b --- /dev/null +++ b/tests/evals/gsm8k/configs/Kimi-K3-pruned75-DSpark-TP4.yaml @@ -0,0 +1,26 @@ +model_name: "mgoin/Kimi-K3-pruned75" +accuracy_threshold: 0.33 +min_acceptance_length: 4.8 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1800 +server_args: >- + --trust-remote-code + --gpu-memory-utilization 0.90 + --tensor-parallel-size 4 + --load-format fastsafetensors + --no-enable-flashinfer-autotune + --max-model-len 1048576 + --max-num-batched-tokens 16384 + --max-num-seqs 128 + --kv-cache-dtype fp8 + --attention-config '{"use_prefill_query_quantization":true,"mla_prefill_backend":"flashinfer"}' + --enable-prefix-caching + --enable-auto-tool-choice + --tool-call-parser kimi_k3 + --reasoning-parser kimi_k3 + --speculative-config '{"method":"dspark", + "model":"RedHatAI/Kimi-K3-speculator.dspark", + "attention_backend":"FLASH_ATTN","num_speculative_tokens":8, + "draft_sample_method":"probabilistic", + "rejection_sample_method":"block"}' diff --git a/tests/evals/gsm8k/configs/models-spec-decode.txt b/tests/evals/gsm8k/configs/models-spec-decode.txt index 05f620435c..427c35d912 100644 --- a/tests/evals/gsm8k/configs/models-spec-decode.txt +++ b/tests/evals/gsm8k/configs/models-spec-decode.txt @@ -1 +1,2 @@ -DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml +DeepSeek-V4-Flash-DSpark-confidence-TEP4.yaml +Kimi-K3-pruned75-DSpark-TP4.yaml diff --git a/vllm/v1/attention/backends/mla/prefill/flashinfer.py b/vllm/v1/attention/backends/mla/prefill/flashinfer.py index 936690ec56..777dd33756 100644 --- a/vllm/v1/attention/backends/mla/prefill/flashinfer.py +++ b/vllm/v1/attention/backends/mla/prefill/flashinfer.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, ClassVar import torch import vllm.envs as envs +from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.v1.attention.backends.mla.prefill.base import ( MLADimensions, MLAPrefillBackend, @@ -161,39 +162,43 @@ class FlashInferPrefillBackend(MLAPrefillBackend): kv_indptr = qo_indptr.clone() assert self._prefill_main is not None - self._prefill_main.plan( - qo_indptr=qo_indptr, - kv_indptr=kv_indptr, - num_qo_heads=num_qo_heads, - num_kv_heads=num_kv_heads, - head_dim_qk=head_dim_qk, - head_dim_vo=head_dim_vo, - causal=True, - sm_scale=global_hyperparameters.sm_scale, - window_left=global_hyperparameters.window_left, - logits_soft_cap=global_hyperparameters.logits_soft_cap, - q_data_type=prefill_metadata.q_data_type, - o_data_type=prefill_metadata.output_dtype, - ) + # FlashInfer's plan() currently copies GPU indptrs to the CPU. + # This will be fixed by https://github.com/vllm-project/vllm/pull/52657. + with gpu_sync_allowed(): + self._prefill_main.plan( + qo_indptr=qo_indptr, + kv_indptr=kv_indptr, + num_qo_heads=num_qo_heads, + num_kv_heads=num_kv_heads, + head_dim_qk=head_dim_qk, + head_dim_vo=head_dim_vo, + causal=True, + sm_scale=global_hyperparameters.sm_scale, + window_left=global_hyperparameters.window_left, + logits_soft_cap=global_hyperparameters.logits_soft_cap, + q_data_type=prefill_metadata.q_data_type, + o_data_type=prefill_metadata.output_dtype, + ) if has_context: chunked_context = prefill_metadata.chunked_context assert chunked_context is not None for chunk in chunked_context.chunks: - self._prefill_chunks[chunk.index].plan( - qo_indptr=chunk.query_start_loc, - kv_indptr=chunk.cu_seq_lens, - num_qo_heads=num_qo_heads, - num_kv_heads=num_kv_heads, - head_dim_qk=head_dim_qk, - head_dim_vo=head_dim_vo, - causal=False, - sm_scale=global_hyperparameters.sm_scale, - window_left=global_hyperparameters.window_left, - logits_soft_cap=global_hyperparameters.logits_soft_cap, - q_data_type=prefill_metadata.q_data_type, - o_data_type=prefill_metadata.output_dtype, - ) + with gpu_sync_allowed(): + self._prefill_chunks[chunk.index].plan( + qo_indptr=chunk.query_start_loc, + kv_indptr=chunk.cu_seq_lens, + num_qo_heads=num_qo_heads, + num_kv_heads=num_kv_heads, + head_dim_qk=head_dim_qk, + head_dim_vo=head_dim_vo, + causal=False, + sm_scale=global_hyperparameters.sm_scale, + window_left=global_hyperparameters.window_left, + logits_soft_cap=global_hyperparameters.logits_soft_cap, + q_data_type=prefill_metadata.q_data_type, + o_data_type=prefill_metadata.output_dtype, + ) def supports_out(self) -> bool: # Planned with head_dim_vo == v_head_dim, so the output is unpadded.