[Kernel]Add norm fused kernel for Step-5 Preview and Muse Spark Open (#62132)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
+33
@@ -363,6 +363,35 @@ minimax_allreduce_rms_qk(torch::stable::Tensor qkv,
|
||||
int64_t const nranks, double const eps);
|
||||
#endif
|
||||
|
||||
#ifndef USE_ROCM
|
||||
namespace vllm {
|
||||
void fused_qknorm_rope_cache(
|
||||
torch::stable::Tensor& q_out, torch::stable::Tensor& k_out,
|
||||
torch::stable::Tensor& v_out, torch::stable::Tensor& qkv,
|
||||
torch::stable::Tensor& q_weight, torch::stable::Tensor& k_weight,
|
||||
torch::stable::Tensor& cos, torch::stable::Tensor& sin,
|
||||
torch::stable::Tensor& positions, torch::stable::Tensor& slot_mapping,
|
||||
torch::stable::Tensor& k_cache, torch::stable::Tensor& v_cache,
|
||||
int64_t head_dim, int64_t num_q_heads, int64_t num_kv_heads,
|
||||
int64_t rotary_pairs, double epsilon, double norm_weight_bias,
|
||||
const std::string& kv_cache_dtype,
|
||||
std::optional<torch::stable::Tensor> k_scale,
|
||||
std::optional<torch::stable::Tensor> v_scale);
|
||||
} // namespace vllm
|
||||
#endif
|
||||
|
||||
void fused_muse_spark_qknorm_rope_kv_insert(
|
||||
torch::stable::Tensor& qkv, torch::stable::Tensor const& positions,
|
||||
int64_t num_heads, int64_t num_kv_heads, int64_t head_dim, double eps,
|
||||
std::optional<torch::stable::Tensor> cos_sin_cache, int64_t rotary_dim,
|
||||
std::optional<torch::stable::Tensor> slot_mapping,
|
||||
std::optional<torch::stable::Tensor> kv_cache, int64_t block_size,
|
||||
const std::string& kv_cache_dtype,
|
||||
std::optional<torch::stable::Tensor> k_scale,
|
||||
std::optional<torch::stable::Tensor> v_scale,
|
||||
std::optional<torch::stable::Tensor> q_fp8_out,
|
||||
std::optional<torch::stable::Tensor> q_scale);
|
||||
|
||||
// Horizontally-fused MiniMax-M3 QK-norm + partial NeoX RoPE (+ optional KV /
|
||||
// index-cache insert). Dense layer: norm+RoPE only; sparse layer: also packs
|
||||
// the index branch and scatters k/v/index_k into their paged caches.
|
||||
@@ -546,6 +575,10 @@ std::tuple<int64_t, torch::stable::Tensor> allocate_shared_buffer_and_handle(
|
||||
int64_t open_mem_handle(torch::stable::Tensor& mem_handle);
|
||||
void free_shared_buffer(int64_t buffer);
|
||||
|
||||
void muse_spark_mul_tanh_soft_cap(torch::stable::Tensor& out,
|
||||
torch::stable::Tensor& input,
|
||||
double multiplier, double soft_cap_temp);
|
||||
|
||||
// Activation kernels (shared CUDA/ROCm)
|
||||
void silu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
void silu_and_mul_clamp(torch::stable::Tensor& out,
|
||||
|
||||
Reference in New Issue
Block a user