[7/n] Migrate pos_encoding and norm kernels to libtorch stable ABI (continued) (#43209)
Signed-off-by: Mikayla Gawarecki <mikaylagawarecki@gmail.com> Signed-off-by: Chris Leonard <chleonar@redhat.com> Co-authored-by: Mikayla Gawarecki <mikaylagawarecki@gmail.com> Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
co-authored by
Mikayla Gawarecki
Shengqi Chen
parent
54d153637b
commit
a7be0f342d
@@ -58,6 +58,35 @@
|
||||
THO_DISPATCH_SWITCH(TYPE, NAME, \
|
||||
VLLM_STABLE_DISPATCH_CASE_HALF_TYPES(__VA_ARGS__))
|
||||
|
||||
// Quant type dispatch (FP8 + INT8)
|
||||
#ifdef USE_ROCM
|
||||
#define VLLM_STABLE_DISPATCH_CASE_QUANT_TYPES(...) \
|
||||
THO_DISPATCH_CASE(torch::headeronly::ScalarType::Float8_e4m3fn, \
|
||||
__VA_ARGS__) \
|
||||
THO_DISPATCH_CASE(torch::headeronly::ScalarType::Float8_e4m3fnuz, \
|
||||
__VA_ARGS__) \
|
||||
THO_DISPATCH_CASE(torch::headeronly::ScalarType::Char, __VA_ARGS__)
|
||||
#else
|
||||
#define VLLM_STABLE_DISPATCH_CASE_QUANT_TYPES(...) \
|
||||
THO_DISPATCH_CASE(torch::headeronly::ScalarType::Float8_e4m3fn, \
|
||||
__VA_ARGS__) \
|
||||
THO_DISPATCH_CASE(torch::headeronly::ScalarType::Char, __VA_ARGS__)
|
||||
#endif
|
||||
|
||||
#define VLLM_STABLE_DISPATCH_QUANT_TYPES(TYPE, NAME, ...) \
|
||||
THO_DISPATCH_SWITCH(TYPE, NAME, \
|
||||
VLLM_STABLE_DISPATCH_CASE_QUANT_TYPES(__VA_ARGS__))
|
||||
|
||||
// Group size dispatch (pure C++ if/else, no ATen dependency)
|
||||
#define VLLM_STABLE_DISPATCH_GROUP_SIZE(group_size, const_group_size, ...) \
|
||||
if (group_size == 128) { \
|
||||
constexpr int const_group_size = 128; \
|
||||
__VA_ARGS__(); \
|
||||
} else if (group_size == 64) { \
|
||||
constexpr int const_group_size = 64; \
|
||||
__VA_ARGS__(); \
|
||||
}
|
||||
|
||||
// Boolean dispatch
|
||||
#define VLLM_STABLE_DISPATCH_BOOL(expr, const_expr, ...) \
|
||||
if (expr) { \
|
||||
@@ -67,3 +96,56 @@
|
||||
constexpr bool const_expr = false; \
|
||||
__VA_ARGS__(); \
|
||||
}
|
||||
|
||||
// Vec size dispatch (pure C++ switch, no ATen dependency)
|
||||
#define VLLM_STABLE_DISPATCH_VEC_SIZE(VEC_SIZE, ...) \
|
||||
switch (VEC_SIZE) { \
|
||||
case 16: { \
|
||||
constexpr int vec_size = 16; \
|
||||
__VA_ARGS__(); \
|
||||
break; \
|
||||
} \
|
||||
case 8: { \
|
||||
constexpr int vec_size = 8; \
|
||||
__VA_ARGS__(); \
|
||||
break; \
|
||||
} \
|
||||
case 4: { \
|
||||
constexpr int vec_size = 4; \
|
||||
__VA_ARGS__(); \
|
||||
break; \
|
||||
} \
|
||||
case 2: { \
|
||||
constexpr int vec_size = 2; \
|
||||
__VA_ARGS__(); \
|
||||
break; \
|
||||
} \
|
||||
default: { \
|
||||
constexpr int vec_size = 1; \
|
||||
__VA_ARGS__(); \
|
||||
break; \
|
||||
} \
|
||||
}
|
||||
|
||||
// Tensor rank dispatch (2D, 3D, 4D)
|
||||
#define VLLM_STABLE_DISPATCH_RANK234(NUM_DIMS, ...) \
|
||||
switch (NUM_DIMS) { \
|
||||
case 2: { \
|
||||
constexpr int tensor_rank = 2; \
|
||||
__VA_ARGS__(); \
|
||||
break; \
|
||||
} \
|
||||
case 3: { \
|
||||
constexpr int tensor_rank = 3; \
|
||||
__VA_ARGS__(); \
|
||||
break; \
|
||||
} \
|
||||
case 4: { \
|
||||
constexpr int tensor_rank = 4; \
|
||||
__VA_ARGS__(); \
|
||||
break; \
|
||||
} \
|
||||
default: \
|
||||
STD_TORCH_CHECK( \
|
||||
false, "Expects rank 2, 3 or 4 tensors but got unsupported rank"); \
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user