[Attention][MLA] Fuse Kimi-K3 chunked-context K/V packing (#51772)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
63344913a6
commit
903d2efe7e
@@ -25,6 +25,14 @@
|
||||
* matching MLA's _q_scale / _k_scale / _v_scale (the cache latent uses a
|
||||
* separate cache scale). Per-tensor E4M3.
|
||||
*
|
||||
* K/V pack (fused_kimi_k3_mla_kv_concat{,_quant_fp8}):
|
||||
* - k_out[t, h] = [k_nope[t, h] | k_pe[t]], cast to fp8 by the _quant_fp8
|
||||
* variant, which also writes v_fp8[t, h] = cast(v[t, h])
|
||||
* The chunked-context epilogue: no cache insert, no q, no RoPE (the cached
|
||||
* k_pe is already rotated) and no scaling. `k_nope`/`v` are strided views
|
||||
* of one kv_b_proj output and `k_pe` may already be fp8 (a plain fp8 cache is
|
||||
* gathered without dequantizing); the outputs are contiguous.
|
||||
*
|
||||
* fp8_ds_mla (fused_kimi_k3_mla_key_concat_ds_mla_insert):
|
||||
* - full key concat (bf16), and cache insert in DeepSeek's 656-byte
|
||||
* block-scaled layout (NoPE fp8 in 4 tiles of 128 with per-tile dynamic
|
||||
@@ -183,6 +191,36 @@ __device__ __forceinline__ void copyChunk8(void* dst, const scalar_t* src,
|
||||
}
|
||||
}
|
||||
|
||||
// Cast 8 source elements (one uint4 of bf16/fp16) to E4M3 with no scaling,
|
||||
// using the native pairwise converters so the result is bit-identical to
|
||||
// `src.to(torch.float8_e4m3fn)`.
|
||||
template <typename scalar_t>
|
||||
__device__ __forceinline__ void copyChunk8UnitFp8(uint8_t* dst,
|
||||
const scalar_t* src) {
|
||||
#ifndef USE_ROCM
|
||||
uint4 const input = *reinterpret_cast<const uint4*>(src);
|
||||
using Converter = vllm::_typeConvert<scalar_t>;
|
||||
auto const* input2 =
|
||||
reinterpret_cast<typename Converter::packed_hip_type const*>(&input);
|
||||
uint2 output;
|
||||
auto* output2 = reinterpret_cast<__nv_fp8x2_storage_t*>(&output);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
if constexpr (std::is_same_v<scalar_t, c10::BFloat16>) {
|
||||
output2[i] = __nv_cvt_bfloat16raw2_to_fp8x2(
|
||||
static_cast<__nv_bfloat162_raw>(input2[i]), __NV_SATFINITE,
|
||||
__NV_E4M3);
|
||||
} else {
|
||||
output2[i] = __nv_cvt_halfraw2_to_fp8x2(
|
||||
static_cast<__half2_raw>(input2[i]), __NV_SATFINITE, __NV_E4M3);
|
||||
}
|
||||
}
|
||||
*reinterpret_cast<uint2*>(dst) = output;
|
||||
#else
|
||||
copyChunk8<scalar_t, true>(dst, src, 1.0f);
|
||||
#endif
|
||||
}
|
||||
|
||||
// Concat + store one head's full key: dst[e] = [k_nope | k_pe], e in [0, 192).
|
||||
// FP8 dst is byte-addressed; bf16 dst is scalar_t-addressed (dst_elem_size).
|
||||
template <typename scalar_t, bool FP8, bool APPLY_ROPE = false>
|
||||
@@ -202,6 +240,38 @@ __device__ __forceinline__ void writeFullKey(void* dst, const scalar_t* k_nope,
|
||||
}
|
||||
}
|
||||
|
||||
// Unscaled full-key writer for the chunked-context pack, driven by a half warp
|
||||
// (16 lanes x 8 elems covers NoPE 128; 8 of them cover RoPE 64). Unlike
|
||||
// `writeFullKey` there is no scale and no RoPE (a gathered k_pe is already
|
||||
// rotated), and KPE_FP8 handles a k_pe that the gather left in the fp8 cache
|
||||
// layout -- so neither dtype needs an intermediate cast or a broadcast copy.
|
||||
template <typename scalar_t, bool FP8, bool KPE_FP8>
|
||||
__device__ __forceinline__ void writeFullKeyPack(void* dst,
|
||||
const scalar_t* k_nope,
|
||||
const void* k_pe, int laneId) {
|
||||
static_assert(!KPE_FP8 || FP8, "an fp8 k_pe requires an fp8 key output");
|
||||
constexpr int kElemSize = FP8 ? 1 : sizeof(scalar_t);
|
||||
auto* d = reinterpret_cast<uint8_t*>(dst);
|
||||
for (int e = laneId * kVecElems; e < kQkHeadDim; e += 16 * kVecElems) {
|
||||
void* out = d + e * kElemSize;
|
||||
const scalar_t* src = k_nope + e;
|
||||
if (e >= kQkNopeHeadDim) {
|
||||
int const rope_e = e - kQkNopeHeadDim;
|
||||
if constexpr (KPE_FP8) {
|
||||
*reinterpret_cast<uint2*>(out) = *reinterpret_cast<const uint2*>(
|
||||
reinterpret_cast<const uint8_t*>(k_pe) + rope_e);
|
||||
continue;
|
||||
}
|
||||
src = reinterpret_cast<const scalar_t*>(k_pe) + rope_e;
|
||||
}
|
||||
if constexpr (FP8) {
|
||||
copyChunk8UnitFp8(reinterpret_cast<uint8_t*>(out), src);
|
||||
} else {
|
||||
*reinterpret_cast<uint4*>(out) = *reinterpret_cast<const uint4*>(src);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Store a prefill query, rotating only q[..., 128:192]. For bf16 dst may alias
|
||||
// q (in-place); fp8 writes the quantized query directly to its output.
|
||||
template <typename scalar_t, bool FP8, bool APPLY_ROPE = false>
|
||||
@@ -457,6 +527,72 @@ __global__ void fusedKimiK3MLAQKVQuantKVCacheFp8Kernel(
|
||||
#endif
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// K/V pack variant (chunked context): concatenate the strided kv_b_proj output
|
||||
// with k_pe into a contiguous row-major key, casting to E4M3 (and casting V
|
||||
// into its own contiguous output) when FP8. Unlike the other kernels this one
|
||||
// has no per-token cache slot, so a warp handles two (token, head) rows -- 16
|
||||
// lanes each -- and the grid is capped so a long context becomes a grid-stride
|
||||
// loop instead of a huge launch.
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
template <typename scalar_t, bool FP8, bool KPE_FP8>
|
||||
__global__ void fusedKimiK3MLAKVConcatPackKernel(
|
||||
const scalar_t* __restrict__ k_nope, int64_t const kn_tok_stride,
|
||||
int64_t const kn_head_stride, const void* __restrict__ k_pe,
|
||||
int64_t const k_pe_tok_stride, const scalar_t* __restrict__ v,
|
||||
int64_t const v_tok_stride, int64_t const v_head_stride,
|
||||
void* __restrict__ k_out, int64_t const ko_tok_stride,
|
||||
int64_t const ko_head_stride, uint8_t* __restrict__ v_fp8,
|
||||
int64_t const vo_tok_stride, int64_t const vo_head_stride,
|
||||
int const num_tokens, int const num_heads) {
|
||||
int const warpsPerBlock = blockDim.x / 32;
|
||||
int const laneId = threadIdx.x % 16;
|
||||
int const rowInWarp = (threadIdx.x % 32) / 16;
|
||||
int64_t const globalWarpIdx =
|
||||
static_cast<int64_t>(blockIdx.x) * warpsPerBlock + threadIdx.x / 32;
|
||||
int64_t const totalRows = static_cast<int64_t>(num_tokens) * num_heads;
|
||||
if (globalWarpIdx * 2 >= totalRows) return;
|
||||
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
|
||||
constexpr int kOutElemSize = FP8 ? 1 : sizeof(scalar_t);
|
||||
int64_t const gridStride =
|
||||
static_cast<int64_t>(gridDim.x) * warpsPerBlock * 2;
|
||||
for (int64_t row = globalWarpIdx * 2 + rowInWarp; row < totalRows;
|
||||
row += gridStride) {
|
||||
int const tokenIdx = static_cast<int>(row / num_heads);
|
||||
int const headIdx = static_cast<int>(row % num_heads);
|
||||
const void* pe;
|
||||
if constexpr (KPE_FP8) {
|
||||
pe = reinterpret_cast<const uint8_t*>(k_pe) + tokenIdx * k_pe_tok_stride;
|
||||
} else {
|
||||
pe = reinterpret_cast<const scalar_t*>(k_pe) + tokenIdx * k_pe_tok_stride;
|
||||
}
|
||||
writeFullKeyPack<scalar_t, FP8, KPE_FP8>(
|
||||
reinterpret_cast<uint8_t*>(k_out) +
|
||||
(tokenIdx * ko_tok_stride + headIdx * ko_head_stride) *
|
||||
kOutElemSize,
|
||||
k_nope + tokenIdx * kn_tok_stride + headIdx * kn_head_stride, pe,
|
||||
laneId);
|
||||
|
||||
// bf16 V needs no cast, so it stays a strided view of the kv_b_proj output.
|
||||
if constexpr (FP8) {
|
||||
const scalar_t* vh =
|
||||
v + tokenIdx * v_tok_stride + headIdx * v_head_stride;
|
||||
uint8_t* vo = v_fp8 + tokenIdx * vo_tok_stride + headIdx * vo_head_stride;
|
||||
for (int e = laneId * kVecElems; e < kVHeadDim; e += 16 * kVecElems) {
|
||||
copyChunk8UnitFp8(vo + e, vh + e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────────────
|
||||
// ds_mla variant: concat full key (bf16) + fp8_ds_mla latent cache insert
|
||||
//
|
||||
@@ -648,16 +784,20 @@ __global__ void fusedKimiK3MLADecodeQConcatDsMlaKernel(
|
||||
#endif
|
||||
}
|
||||
|
||||
// PDL-aware launch of a (token, num_heads + 1)-warp grid.
|
||||
// PDL-aware launch of a (token, slots_per_token)-row grid, `rows_per_warp` rows
|
||||
// to a warp. `max_blocks > 0` caps the grid, leaving the kernel's grid-stride
|
||||
// loop to cover the remaining rows.
|
||||
template <typename KernelT, typename... Args>
|
||||
static void launchPdl(KernelT kernel, int num_tokens, int num_heads,
|
||||
cudaStream_t stream, Args... args) {
|
||||
static void launchPdlSlots(KernelT kernel, int num_tokens, int slots_per_token,
|
||||
int rows_per_warp, int max_blocks,
|
||||
cudaStream_t stream, Args... args) {
|
||||
constexpr int kBlockSize = 256;
|
||||
constexpr int kWarpsPerBlock = kBlockSize / 32;
|
||||
int64_t const total_warps =
|
||||
static_cast<int64_t>(num_tokens) * (num_heads + 1);
|
||||
int const grid =
|
||||
int64_t const total_rows = static_cast<int64_t>(num_tokens) * slots_per_token;
|
||||
int64_t const total_warps = (total_rows + rows_per_warp - 1) / rows_per_warp;
|
||||
int grid =
|
||||
static_cast<int>((total_warps + kWarpsPerBlock - 1) / kWarpsPerBlock);
|
||||
if (max_blocks > 0 && grid > max_blocks) grid = max_blocks;
|
||||
#ifndef USE_ROCM
|
||||
static int const sm_version = getSMVersion();
|
||||
cudaLaunchConfig_t config;
|
||||
@@ -681,6 +821,13 @@ static void launchPdl(KernelT kernel, int num_tokens, int num_heads,
|
||||
#endif
|
||||
}
|
||||
|
||||
// Cache-inserting kernels use one extra warp per token for the cache slot.
|
||||
template <typename KernelT, typename... Args>
|
||||
static void launchPdl(KernelT kernel, int num_tokens, int num_heads,
|
||||
cudaStream_t stream, Args... args) {
|
||||
launchPdlSlots(kernel, num_tokens, num_heads + 1, 1, 0, stream, args...);
|
||||
}
|
||||
|
||||
void checkBfloat16Support(torch::headeronly::ScalarType dtype) {
|
||||
#ifndef USE_ROCM
|
||||
if (dtype == torch::headeronly::ScalarType::BFloat16) {
|
||||
@@ -913,6 +1060,128 @@ void fused_kimi_k3_mla_key_concat_ds_mla_insert(
|
||||
});
|
||||
}
|
||||
|
||||
namespace {
|
||||
// Shared checks for the chunked-context K/V pack ops. `k_nope` / `v` are
|
||||
// strided views of one kv_b_proj output and `k_pe` is a strided view of the
|
||||
// gathered context workspace, so only a unit last-dim stride (not full
|
||||
// contiguity) is required of the inputs. Returns true when `k_pe` is still in
|
||||
// the fp8 cache layout.
|
||||
bool check_kv_concat_inputs(torch::stable::Tensor const& k_nope,
|
||||
torch::stable::Tensor const& k_pe,
|
||||
torch::stable::Tensor const& k_out,
|
||||
torch::headeronly::ScalarType out_dtype) {
|
||||
using torch::headeronly::ScalarType;
|
||||
ScalarType const dt = k_nope.scalar_type();
|
||||
STD_TORCH_CHECK(k_nope.device().is_cuda() && k_nope.dim() == 3 &&
|
||||
k_nope.size(2) == 128 && k_nope.stride(2) == 1,
|
||||
"k_nope must be CUDA [T, H, 128] with unit last stride");
|
||||
bool const k_pe_is_fp8 = k_pe.scalar_type() == ScalarType::Float8_e4m3fn;
|
||||
STD_TORCH_CHECK(
|
||||
k_pe.device() == k_nope.device() && k_pe.dim() == 2 &&
|
||||
k_pe.size(1) == 64 && k_pe.stride(1) == 1 &&
|
||||
(k_pe.scalar_type() == dt ||
|
||||
(k_pe_is_fp8 && out_dtype == ScalarType::Float8_e4m3fn)),
|
||||
"k_pe must be CUDA [T, 64] with unit last-dim stride and use the input "
|
||||
"dtype, or float8_e4m3fn when the key output is fp8");
|
||||
STD_TORCH_CHECK(k_out.device() == k_nope.device() && k_out.is_contiguous() &&
|
||||
k_out.scalar_type() == out_dtype && k_out.dim() == 3 &&
|
||||
k_out.size(2) == 192 && k_out.size(0) == k_nope.size(0) &&
|
||||
k_out.size(1) == k_nope.size(1) &&
|
||||
k_pe.size(0) == k_nope.size(0),
|
||||
"k_out must be a contiguous CUDA [T, H, 192] tensor of the "
|
||||
"output dtype matching k_nope's token and head dimensions");
|
||||
vllm::kimi_k3_fused_ops::checkBfloat16Support(dt);
|
||||
return k_pe_is_fp8;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void fused_kimi_k3_mla_kv_concat(
|
||||
torch::stable::Tensor const& k_nope, // [T, H, 128] bf16/fp16
|
||||
torch::stable::Tensor const& k_pe, // [T, 64] same dtype as k_nope
|
||||
torch::stable::Tensor& k_out) { // [T, H, 192] same dtype, written
|
||||
namespace kk3 = vllm::kimi_k3_fused_ops;
|
||||
torch::headeronly::ScalarType const dt = k_nope.scalar_type();
|
||||
check_kv_concat_inputs(k_nope, k_pe, k_out, dt);
|
||||
|
||||
int const num_tokens = static_cast<int>(k_nope.size(0));
|
||||
int const num_heads = static_cast<int>(k_nope.size(1));
|
||||
if (num_tokens == 0) return;
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
k_nope.get_device_index());
|
||||
const cudaStream_t stream =
|
||||
get_current_cuda_stream(k_nope.get_device_index());
|
||||
|
||||
VLLM_STABLE_DISPATCH_HALF_TYPES(dt, "fused_kimi_k3_mla_kv_concat", [&] {
|
||||
kk3::launchPdlSlots(
|
||||
kk3::fusedKimiK3MLAKVConcatPackKernel<scalar_t, false, false>,
|
||||
num_tokens, num_heads, 2, get_device_prop()->multiProcessorCount * 8,
|
||||
stream, reinterpret_cast<scalar_t const*>(k_nope.const_data_ptr()),
|
||||
k_nope.stride(0), k_nope.stride(1), k_pe.const_data_ptr(),
|
||||
k_pe.stride(0), static_cast<scalar_t const*>(nullptr), int64_t{0},
|
||||
int64_t{0}, k_out.mutable_data_ptr(), k_out.stride(0), k_out.stride(1),
|
||||
static_cast<uint8_t*>(nullptr), int64_t{0}, int64_t{0}, num_tokens,
|
||||
num_heads);
|
||||
});
|
||||
}
|
||||
|
||||
void fused_kimi_k3_mla_kv_concat_quant_fp8(
|
||||
torch::stable::Tensor const& k_nope, // [T, H, 128] bf16/fp16
|
||||
torch::stable::Tensor const& k_pe, // [T, 64] bf16/fp16/fp8
|
||||
torch::stable::Tensor const& v, // [T, H, 128] bf16/fp16
|
||||
torch::stable::Tensor& k_fp8, // [T, H, 192] fp8, written
|
||||
torch::stable::Tensor& v_fp8) { // [T, H, 128] fp8, written
|
||||
using torch::headeronly::ScalarType;
|
||||
namespace kk3 = vllm::kimi_k3_fused_ops;
|
||||
ScalarType const dt = k_nope.scalar_type();
|
||||
bool const k_pe_is_fp8 =
|
||||
check_kv_concat_inputs(k_nope, k_pe, k_fp8, ScalarType::Float8_e4m3fn);
|
||||
STD_TORCH_CHECK(v.device() == k_nope.device() && v.scalar_type() == dt &&
|
||||
v.dim() == 3 && v.size(2) == 128 && v.stride(2) == 1 &&
|
||||
v.size(0) == k_nope.size(0) &&
|
||||
v.size(1) == k_nope.size(1),
|
||||
"v must match k_nope's dtype and token/head dimensions and "
|
||||
"have shape [T, H, 128] with unit last stride");
|
||||
STD_TORCH_CHECK(v_fp8.device() == k_nope.device() && v_fp8.is_contiguous() &&
|
||||
v_fp8.scalar_type() == ScalarType::Float8_e4m3fn &&
|
||||
v_fp8.dim() == 3 && v_fp8.size(2) == 128 &&
|
||||
v_fp8.size(0) == k_nope.size(0) &&
|
||||
v_fp8.size(1) == k_nope.size(1),
|
||||
"v_fp8 must be a contiguous CUDA [T, H, 128] fp8 tensor "
|
||||
"matching k_nope's token and head dimensions");
|
||||
|
||||
int const num_tokens = static_cast<int>(k_nope.size(0));
|
||||
int const num_heads = static_cast<int>(k_nope.size(1));
|
||||
if (num_tokens == 0) return;
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
k_nope.get_device_index());
|
||||
const cudaStream_t stream =
|
||||
get_current_cuda_stream(k_nope.get_device_index());
|
||||
|
||||
VLLM_STABLE_DISPATCH_HALF_TYPES(
|
||||
dt, "fused_kimi_k3_mla_kv_concat_quant_fp8", [&] {
|
||||
auto launch = [&](auto kernel) {
|
||||
kk3::launchPdlSlots(
|
||||
kernel, num_tokens, num_heads, 2,
|
||||
get_device_prop()->multiProcessorCount * 8, stream,
|
||||
reinterpret_cast<scalar_t const*>(k_nope.const_data_ptr()),
|
||||
k_nope.stride(0), k_nope.stride(1), k_pe.const_data_ptr(),
|
||||
k_pe.stride(0),
|
||||
reinterpret_cast<scalar_t const*>(v.const_data_ptr()),
|
||||
v.stride(0), v.stride(1), k_fp8.mutable_data_ptr(),
|
||||
k_fp8.stride(0), k_fp8.stride(1),
|
||||
reinterpret_cast<uint8_t*>(v_fp8.mutable_data_ptr()),
|
||||
v_fp8.stride(0), v_fp8.stride(1), num_tokens, num_heads);
|
||||
};
|
||||
if (k_pe_is_fp8) {
|
||||
launch(kk3::fusedKimiK3MLAKVConcatPackKernel<scalar_t, true, true>);
|
||||
} else {
|
||||
launch(kk3::fusedKimiK3MLAKVConcatPackKernel<scalar_t, true, false>);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert(
|
||||
torch::stable::Tensor const& q, // [Tp, H, 192] bf16
|
||||
torch::stable::Tensor const& k_nope, // [Tp, H, 128] bf16
|
||||
|
||||
@@ -300,6 +300,16 @@ void fused_kimi_k3_mla_key_concat_ds_mla_insert(
|
||||
std::optional<torch::stable::Tensor> position_ids,
|
||||
std::optional<torch::stable::Tensor> cos_sin_cache);
|
||||
|
||||
void fused_kimi_k3_mla_kv_concat(torch::stable::Tensor const& k_nope,
|
||||
torch::stable::Tensor const& k_pe,
|
||||
torch::stable::Tensor& k_out);
|
||||
|
||||
void fused_kimi_k3_mla_kv_concat_quant_fp8(torch::stable::Tensor const& k_nope,
|
||||
torch::stable::Tensor const& k_pe,
|
||||
torch::stable::Tensor const& v,
|
||||
torch::stable::Tensor& k_fp8,
|
||||
torch::stable::Tensor& v_fp8);
|
||||
|
||||
void fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert(
|
||||
torch::stable::Tensor const& q, torch::stable::Tensor const& k_nope,
|
||||
torch::stable::Tensor const& k_pe, torch::stable::Tensor const& kv_c_normed,
|
||||
|
||||
@@ -466,6 +466,13 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"Tensor! k_out, Tensor! k_cache, Tensor slot_mapping, "
|
||||
"int cache_block_size, Tensor? position_ids=None, "
|
||||
"Tensor? cos_sin_cache=None) -> ()");
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_kv_concat(Tensor k_nope, Tensor k_pe, Tensor! k_out) "
|
||||
"-> ()");
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_kv_concat_quant_fp8("
|
||||
"Tensor k_nope, Tensor k_pe, Tensor v, Tensor! k_fp8, Tensor! v_fp8) "
|
||||
"-> ()");
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert("
|
||||
"Tensor q, Tensor k_nope, Tensor k_pe, Tensor kv_c_normed, Tensor v, "
|
||||
@@ -769,6 +776,10 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
TORCH_BOX(&fused_kimi_k3_mla_key_concat_kv_cache_insert));
|
||||
ops.impl("fused_kimi_k3_mla_key_concat_ds_mla_insert",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_key_concat_ds_mla_insert));
|
||||
ops.impl("fused_kimi_k3_mla_kv_concat",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_kv_concat));
|
||||
ops.impl("fused_kimi_k3_mla_kv_concat_quant_fp8",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_kv_concat_quant_fp8));
|
||||
ops.impl("fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert));
|
||||
ops.impl("fused_kimi_k3_mla_decode_q_concat_kv_cache_insert",
|
||||
|
||||
Reference in New Issue
Block a user