[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
parent
6f00a1ae3b
commit
7c6729b769
@@ -13,10 +13,18 @@ namespace vllm {
|
||||
namespace fp8 {
|
||||
#ifdef ENABLE_FP8
|
||||
|
||||
// Unspecialized conversions are a compile error: the old passthrough
|
||||
// (`return x;`) silently skipped fp8 encoding for any (Tout, Tin) pair
|
||||
// without a specialization below (e.g. the torch stable-ABI scalar types),
|
||||
// corrupting quantized data with no runtime signal.
|
||||
template <typename>
|
||||
inline constexpr bool _no_conversion_specialization = false;
|
||||
|
||||
template <typename Tout, typename Tin>
|
||||
__inline__ __device__ Tout vec_conversion(
|
||||
const Tin& x, const __nv_fp8_interpretation_t fp8_type = __NV_E4M3) {
|
||||
return x;
|
||||
static_assert(_no_conversion_specialization<Tin>,
|
||||
"no vec_conversion specialization for this (Tout, Tin) pair");
|
||||
}
|
||||
|
||||
// float -> c10::Float8_e4m3fn
|
||||
@@ -301,7 +309,9 @@ __inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, Float8_>(
|
||||
template <typename Tout, typename Tin>
|
||||
__inline__ __device__ Tout scaled_vec_conversion(
|
||||
const Tin& x, const float scale, const __nv_fp8_interpretation_t fp8_type) {
|
||||
return x;
|
||||
static_assert(
|
||||
_no_conversion_specialization<Tin>,
|
||||
"no scaled_vec_conversion specialization for this (Tout, Tin) pair");
|
||||
}
|
||||
|
||||
// fp8 -> half
|
||||
@@ -492,6 +502,25 @@ __inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, __nv_bfloat16>(
|
||||
__builtin_unreachable(); // Suppress missing return statement warning
|
||||
}
|
||||
|
||||
// torch stable-ABI (headeronly) scalar types delegate to the CUDA-native
|
||||
// conversions, so libtorch_stable kernels dispatched on c10::BFloat16 /
|
||||
// c10::Half quantize correctly without manual casts.
|
||||
template <>
|
||||
__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, c10::BFloat16>(
|
||||
const c10::BFloat16& a, const float scale,
|
||||
const __nv_fp8_interpretation_t fp8_type) {
|
||||
return scaled_vec_conversion<uint8_t, __nv_bfloat16>(
|
||||
reinterpret_cast<const __nv_bfloat16&>(a), scale, fp8_type);
|
||||
}
|
||||
|
||||
template <>
|
||||
__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, c10::Half>(
|
||||
const c10::Half& a, const float scale,
|
||||
const __nv_fp8_interpretation_t fp8_type) {
|
||||
return scaled_vec_conversion<uint8_t, uint16_t>(
|
||||
reinterpret_cast<const uint16_t&>(a), scale, fp8_type);
|
||||
}
|
||||
|
||||
// float -> fp8
|
||||
template <>
|
||||
__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, float>(
|
||||
|
||||
Reference in New Issue
Block a user