[Model] Add DeepSeek-V4 CPU backend (#55355)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-09-10 14:14:48 +08:00
committed by GitHub
parent 49632fb3ac
commit c3ccc0e957
47 changed files with 9564 additions and 627 deletions
+213 -16
View File
@@ -66,18 +66,18 @@ at::Tensor convert_weight_packed(at::Tensor& weight);
at::Tensor convert_scale_packed(at::Tensor& scale);
at::Tensor fused_experts_cpu(
at::Tensor& hidden_states, at::Tensor& w1, at::Tensor& w2,
at::Tensor& topk_weights, at::Tensor& topk_ids, bool inplace,
int64_t moe_comp_method, const std::optional<at::Tensor>& w1_scale,
const std::optional<at::Tensor>& w2_scale,
const std::optional<at::Tensor>& w1_zero,
const std::optional<at::Tensor>& w2_zero,
const std::optional<std::vector<int64_t>> block_size,
const std::optional<at::Tensor>& w1_bias,
const std::optional<at::Tensor>& w2_bias,
const std::optional<double>& alpha, const std::optional<double>& limit,
bool is_vnni);
void fused_experts_cpu(at::Tensor& out, at::Tensor& hidden_states,
at::Tensor& w1, at::Tensor& w2, at::Tensor& topk_weights,
at::Tensor& topk_ids, int64_t moe_comp_method,
const std::optional<at::Tensor>& w1_scale,
const std::optional<at::Tensor>& w2_scale,
const std::optional<at::Tensor>& w1_zero,
const std::optional<at::Tensor>& w2_zero,
const std::optional<std::vector<int64_t>> block_size,
const std::optional<at::Tensor>& w1_bias,
const std::optional<at::Tensor>& w2_bias,
const std::optional<double>& alpha,
const std::optional<double>& limit, bool is_vnni);
at::Tensor int8_scaled_mm_with_quant(at::Tensor& mat1, at::Tensor& mat2,
at::Tensor& scales2,
@@ -124,6 +124,99 @@ void concat_and_cache_mla_cpu(const at::Tensor& kv_c_normed,
const at::Tensor& k_pe, at::Tensor& kv_cache,
const at::Tensor& slot_mapping);
// Adapted from sglang: DeepSeek-V4 mHC gating kernels
std::tuple<at::Tensor, at::Tensor, at::Tensor> hc_pre_fused_cpu(
at::Tensor& x, at::Tensor& hc_fn, at::Tensor& hc_scale, at::Tensor& hc_base,
int64_t hc_mult, int64_t sinkhorn_iters, double rms_eps, double hc_eps);
at::Tensor hc_post_fused_cpu(at::Tensor& x, at::Tensor& residual,
at::Tensor& post, at::Tensor& comb);
at::Tensor hc_head_fused_cpu(at::Tensor& x, at::Tensor& hc_fn,
at::Tensor& hc_scale, at::Tensor& hc_base,
double hc_eps, double norm_eps);
// Adapted from sglang: DeepSeek-V4 fp8_ds_mla cache-write kernel
at::Tensor fused_qnorm_rope_kv_insert_cpu(at::Tensor& q, at::Tensor& kv,
at::Tensor& positions,
at::Tensor& swa_kv_cache_2d,
at::Tensor& slot_mapping,
at::Tensor& cos_sin_cache,
int64_t q_head_padded, double eps,
int64_t cache_block_size);
// vLLM-native: DeepSeek-V4 sparse MQA attention kernel for the fp8_ds_mla
// cache layout (SWA window + optional compressed top-k index sets).
void flash_mla_with_kvcache_cpu(
at::Tensor& out, at::Tensor& q, at::Tensor& window_cache_2d,
at::Tensor& window_slots, int64_t window_block_size,
at::Tensor& compressed_cache_2d, at::Tensor& compressed_slots,
int64_t compressed_block_size, at::Tensor& attn_sink, double scale);
// Adapted from sglang: DeepSeek-V4 compressor state-cache write + fused
// compress+RMSNorm+quant+RoPE+store kernels.
void save_partial_states_cpu(at::Tensor& kv, at::Tensor& score, at::Tensor& ape,
at::Tensor& positions, at::Tensor& state_cache,
at::Tensor& slot_mapping);
void compress_norm_rope_store_cpu(
at::Tensor& state_cache, at::Tensor& gather_slots, at::Tensor& positions,
at::Tensor& kv_slot_mapping, at::Tensor& rms_norm_weight,
double rms_norm_eps, at::Tensor& cos_sin_cache, at::Tensor& kv_cache_2d,
int64_t kv_cache_block_size, int64_t compress_ratio);
// head_dim=128 indexer-compressor variant of the above (single fp8 quant
// block + raw fp32 scale -- matches the paged indexer K-cache layout
// fp8_paged_mqa_logits_cpu/topk_transform_512_cpu read).
void compress_norm_rope_store_indexer_cpu(
at::Tensor& state_cache, at::Tensor& gather_slots, at::Tensor& positions,
at::Tensor& kv_slot_mapping, at::Tensor& rms_norm_weight,
double rms_norm_eps, at::Tensor& cos_sin_cache, at::Tensor& kv_cache_2d,
int64_t kv_cache_block_size, int64_t compress_ratio);
// Adapted from sglang: DeepSeek-V4 sparse indexer paged MQA-logits +
// top-512 transform kernels (DECODE path -- reads the paged K-cache
// directly via `page_table`, whole batch in one call).
at::Tensor fp8_paged_mqa_logits_cpu(at::Tensor& q_fp8, at::Tensor& kvcache_fp8,
at::Tensor& weight, at::Tensor& seq_lens,
at::Tensor& page_table, int64_t block_size,
int64_t max_seq_len);
void topk_transform_512_cpu(at::Tensor& scores, at::Tensor& seq_lens,
at::Tensor& page_tables,
at::Tensor& out_page_indices, int64_t page_size,
const std::optional<at::Tensor>& out_raw_indices);
// Adapted from sglang: DeepSeek-V4 MoE routing kernels (flat biased top-k and
// hash-routed-layer top-k).
std::tuple<at::Tensor, at::Tensor> biased_topk_cpu(
at::Tensor& hidden_states, at::Tensor& gating_output,
at::Tensor& correction_bias, int64_t topk, bool renormalize,
std::string scoring_func, int64_t num_fused_shared_experts,
std::optional<double> routed_scaling_factor,
bool apply_routed_scaling_factor_on_output);
std::tuple<at::Tensor, at::Tensor> hash_topk_cpu(
at::Tensor& gating_output, at::Tensor& tid2eid, int64_t topk,
std::string scoring_func, int64_t num_fused_shared_experts,
int64_t num_experts, double routed_scaling_factor);
// vLLM-native: DeepSeek-V4 sparse indexer Q-side RoPE + FP8 quant kernel
// (fp8 path only; MXFP4 stays on triton-cpu).
void fused_indexer_q_rope_quant_cpu(at::Tensor& positions, at::Tensor& index_q,
at::Tensor& index_q_cos_sin_cache,
at::Tensor& index_q_fp8,
at::Tensor& index_weights,
double index_weights_softmax_scale,
double index_weights_head_scale,
at::Tensor& index_weights_out);
// vLLM-native: inverse GPT-J RoPE for DeepSeek-V4 CPU attention's `_o_proj`
// output de-rotation.
at::Tensor inverse_gptj_rope_o_proj_cpu(at::Tensor& o, at::Tensor& positions,
at::Tensor& cos_sin_cache,
int64_t rope_dim);
// Adapted from sglang: INT4 W4A8 kernels
std::tuple<at::Tensor, at::Tensor, at::Tensor> convert_weight_packed_scale_zp(
at::Tensor qweight, // awq: (*, K, N / 8) || gptq: (*, K / 8, N) , int32
@@ -515,13 +608,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
ops.def("convert_scale_packed(Tensor! scale) -> Tensor");
ops.impl("convert_scale_packed", torch::kCPU, &convert_scale_packed);
ops.def(
"fused_experts_cpu(Tensor hidden_states, Tensor w1, Tensor w2, Tensor "
"topk_weights, Tensor topk_ids, bool "
"inplace, int moe_comp_method, Tensor? w1_scale, Tensor? w2_scale, "
"fused_experts_cpu(Tensor(a0!) out, Tensor hidden_states, Tensor w1, "
"Tensor w2, Tensor topk_weights, Tensor topk_ids, "
"int moe_comp_method, Tensor? w1_scale, Tensor? w2_scale, "
"Tensor? w1_zero, Tensor? w2_zero, int[]? block_size, "
"Tensor? w1_bias, Tensor? w2_bias, float? alpha, float? limit, "
"bool is_vnni) -> "
"Tensor");
"()");
ops.impl("fused_experts_cpu", torch::kCPU, &fused_experts_cpu);
ops.def(
"int8_scaled_mm_with_quant(Tensor mat1, Tensor mat2, Tensor scales2, "
@@ -583,6 +676,110 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
"concat_and_cache_mla_cpu(Tensor kv_c_normed, Tensor k_pe, "
"Tensor(a!) kv_cache, Tensor slot_mapping) -> ()");
ops.impl("concat_and_cache_mla_cpu", torch::kCPU, &concat_and_cache_mla_cpu);
// Adapted from sglang: DeepSeek-V4 mHC gating kernels
ops.def(
"hc_pre_fused_cpu(Tensor x, Tensor hc_fn, Tensor hc_scale, "
"Tensor hc_base, int hc_mult, int sinkhorn_iters, float rms_eps, "
"float hc_eps) -> (Tensor, Tensor, Tensor)");
ops.impl("hc_pre_fused_cpu", torch::kCPU, &hc_pre_fused_cpu);
ops.def(
"hc_post_fused_cpu(Tensor x, Tensor residual, Tensor post, "
"Tensor comb) -> Tensor");
ops.impl("hc_post_fused_cpu", torch::kCPU, &hc_post_fused_cpu);
ops.def(
"hc_head_fused_cpu(Tensor x, Tensor hc_fn, Tensor "
"hc_scale, Tensor hc_base, float hc_eps, float norm_eps) -> "
"Tensor");
ops.impl("hc_head_fused_cpu", torch::kCPU, &hc_head_fused_cpu);
// Adapted from sglang: DeepSeek-V4 fp8_ds_mla cache-write kernel
ops.def(
"fused_qnorm_rope_kv_insert_cpu(Tensor q, Tensor kv, Tensor positions, "
"Tensor(a!) swa_kv_cache_2d, Tensor slot_mapping, Tensor "
"cos_sin_cache, int q_head_padded, float eps, int cache_block_size) "
"-> Tensor");
ops.impl("fused_qnorm_rope_kv_insert_cpu", torch::kCPU,
&fused_qnorm_rope_kv_insert_cpu);
// vLLM-native: DeepSeek-V4 sparse MQA attention kernel
ops.def(
"flash_mla_with_kvcache_cpu(Tensor(a!) out, Tensor q, Tensor "
"window_cache_2d, Tensor window_slots, int window_block_size, Tensor "
"compressed_cache_2d, Tensor compressed_slots, int "
"compressed_block_size, Tensor attn_sink, float scale) -> ()");
ops.impl("flash_mla_with_kvcache_cpu", torch::kCPU,
&flash_mla_with_kvcache_cpu);
// Adapted from sglang: DeepSeek-V4 compressor kernels
ops.def(
"save_partial_states_cpu(Tensor kv, Tensor score, Tensor ape, "
"Tensor positions, Tensor(a!) state_cache, Tensor slot_mapping) "
"-> ()");
ops.impl("save_partial_states_cpu", torch::kCPU, &save_partial_states_cpu);
ops.def(
"compress_norm_rope_store_cpu(Tensor state_cache, Tensor "
"gather_slots, Tensor positions, Tensor kv_slot_mapping, Tensor "
"rms_norm_weight, float rms_norm_eps, Tensor cos_sin_cache, "
"Tensor(a!) kv_cache_2d, int kv_cache_block_size, int compress_ratio) "
"-> ()");
ops.impl("compress_norm_rope_store_cpu", torch::kCPU,
&compress_norm_rope_store_cpu);
ops.def(
"compress_norm_rope_store_indexer_cpu(Tensor state_cache, Tensor "
"gather_slots, Tensor positions, Tensor kv_slot_mapping, Tensor "
"rms_norm_weight, float rms_norm_eps, Tensor cos_sin_cache, "
"Tensor(a!) kv_cache_2d, int kv_cache_block_size, int compress_ratio) "
"-> ()");
ops.impl("compress_norm_rope_store_indexer_cpu", torch::kCPU,
&compress_norm_rope_store_indexer_cpu);
// Adapted from sglang: DeepSeek-V4 sparse indexer paged MQA-logits kernel
// (DECODE path)
ops.def(
"fp8_paged_mqa_logits_cpu(Tensor q_fp8, Tensor kvcache_fp8, Tensor "
"weight, Tensor seq_lens, Tensor page_table, int block_size, int "
"max_seq_len) -> Tensor");
ops.impl("fp8_paged_mqa_logits_cpu", torch::kCPU, &fp8_paged_mqa_logits_cpu);
ops.def(
"topk_transform_512_cpu(Tensor scores, Tensor seq_lens, Tensor "
"page_tables, Tensor(a!) out_page_indices, int page_size, Tensor(b!)? "
"out_raw_indices) -> ()");
ops.impl("topk_transform_512_cpu", torch::kCPU, &topk_transform_512_cpu);
ops.def(
"fused_indexer_q_rope_quant_cpu(Tensor positions, Tensor index_q, "
"Tensor index_q_cos_sin_cache, Tensor(a!) index_q_fp8, Tensor "
"index_weights, float index_weights_softmax_scale, float "
"index_weights_head_scale, Tensor(b!) index_weights_out) -> ()");
ops.impl("fused_indexer_q_rope_quant_cpu", torch::kCPU,
&fused_indexer_q_rope_quant_cpu);
// Adapted from sglang: DeepSeek-V4 MoE routing kernels
ops.def(
"biased_topk_cpu(Tensor hidden_states, Tensor gating_output, Tensor "
"correction_bias, int topk, bool renormalize, str scoring_func, int "
"num_fused_shared_experts, float? routed_scaling_factor, bool "
"apply_routed_scaling_factor_on_output) -> (Tensor, Tensor)");
ops.impl("biased_topk_cpu", torch::kCPU, &biased_topk_cpu);
ops.def(
"hash_topk_cpu(Tensor gating_output, Tensor tid2eid, int topk, str "
"scoring_func, int num_fused_shared_experts, int num_experts, float "
"routed_scaling_factor) -> (Tensor, Tensor)");
ops.impl("hash_topk_cpu", torch::kCPU, &hash_topk_cpu);
// vLLM-native: inverse GPT-J RoPE for _o_proj's output de-rotation
ops.def(
"inverse_gptj_rope_o_proj_cpu(Tensor o, Tensor positions, Tensor "
"cos_sin_cache, int rope_dim) -> Tensor");
ops.impl("inverse_gptj_rope_o_proj_cpu", torch::kCPU,
&inverse_gptj_rope_o_proj_cpu);
#endif
#if (defined(__AVX512BF16__) && defined(__AVX512F__) && \