[Migration] Migrate GGUF quantization support to plugin (#39612)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py
@@ -40,3 +40,17 @@ steps:
|
||||
- pytest -v -s plugins_tests/test_oot_registration_online.py # it needs a clean process
|
||||
- pytest -v -s plugins_tests/test_oot_registration_offline.py # it needs a clean process
|
||||
- pytest -v -s plugins_tests/lora_resolvers # unit tests for in-tree lora resolver plugins
|
||||
|
||||
|
||||
- label: GGUF Plugin
|
||||
key: gguf-plugin
|
||||
device: h200_18gb
|
||||
timeout_in_minutes: 30
|
||||
soft_fail: true
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/layers/quantization
|
||||
- tests/plugins_tests/test_gguf_plugin.py
|
||||
commands:
|
||||
- pip install "vllm-gguf-plugin >= 0.0.2"
|
||||
- pytest -v -s plugins_tests/gguf
|
||||
|
||||
@@ -21,7 +21,6 @@ updates:
|
||||
- dependency-name: "torchvision"
|
||||
- dependency-name: "xformers"
|
||||
- dependency-name: "lm-format-enforcer"
|
||||
- dependency-name: "gguf"
|
||||
- dependency-name: "compressed-tensors"
|
||||
- dependency-name: "ray[cgraph]" # Ray Compiled Graph
|
||||
- dependency-name: "lm-eval"
|
||||
|
||||
@@ -21,7 +21,7 @@ repos:
|
||||
rev: v21.1.2
|
||||
hooks:
|
||||
- id: clang-format
|
||||
exclude: 'csrc/(libtorch_stable/moe/topk_softmax_kernels.cu|libtorch_stable/quantization/gguf/(ggml-common.h|dequantize.cuh|vecdotq.cuh|mmq.cuh|mmvq.cuh))|vllm/third_party/.*'
|
||||
exclude: 'csrc/libtorch_stable/moe/topk_softmax_kernels.cu|vllm/third_party/.*'
|
||||
types_or: [c++, cuda]
|
||||
args: [--style=file, --verbose]
|
||||
- repo: https://github.com/DavidAnson/markdownlint-cli2
|
||||
|
||||
@@ -433,7 +433,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
|
||||
"csrc/libtorch_stable/quantization/w8a8/int8/per_token_group_quant.cu"
|
||||
"csrc/libtorch_stable/permute_cols.cu"
|
||||
"csrc/libtorch_stable/quantization/gptq/q_gemm.cu"
|
||||
"csrc/libtorch_stable/quantization/gguf/gguf_kernel.cu"
|
||||
"csrc/libtorch_stable/pos_encoding_kernels.cu"
|
||||
"csrc/libtorch_stable/fused_qknorm_rope_kernel.cu"
|
||||
"csrc/libtorch_stable/layernorm_kernels.cu"
|
||||
|
||||
@@ -397,35 +397,6 @@ torch::stable::Tensor gptq_gemm(torch::stable::Tensor a,
|
||||
void gptq_shuffle(torch::stable::Tensor q_weight, torch::stable::Tensor q_perm,
|
||||
int64_t bit);
|
||||
|
||||
// GGML kernels (shared CUDA/ROCm)
|
||||
torch::stable::Tensor ggml_dequantize(
|
||||
torch::stable::Tensor W, int64_t type, int64_t m, int64_t n,
|
||||
std::optional<torch::headeronly::ScalarType> const& dtype);
|
||||
|
||||
torch::stable::Tensor ggml_mul_mat_vec_a8(torch::stable::Tensor W,
|
||||
torch::stable::Tensor X, int64_t type,
|
||||
int64_t row);
|
||||
|
||||
torch::stable::Tensor ggml_mul_mat_a8(torch::stable::Tensor W,
|
||||
torch::stable::Tensor X, int64_t type,
|
||||
int64_t row);
|
||||
|
||||
torch::stable::Tensor ggml_moe_a8(torch::stable::Tensor X,
|
||||
torch::stable::Tensor W,
|
||||
torch::stable::Tensor sorted_token_ids,
|
||||
torch::stable::Tensor expert_ids,
|
||||
torch::stable::Tensor num_tokens_post_padded,
|
||||
int64_t type, int64_t row, int64_t top_k,
|
||||
int64_t tokens);
|
||||
|
||||
torch::stable::Tensor ggml_moe_a8_vec(torch::stable::Tensor X,
|
||||
torch::stable::Tensor W,
|
||||
torch::stable::Tensor topk_ids,
|
||||
int64_t top_k, int64_t type, int64_t row,
|
||||
int64_t tokens);
|
||||
|
||||
int64_t ggml_moe_get_block_size(int64_t type);
|
||||
|
||||
void paged_attention_v1(
|
||||
torch::stable::Tensor& out, torch::stable::Tensor& query,
|
||||
torch::stable::Tensor& key_cache, torch::stable::Tensor& value_cache,
|
||||
|
||||
@@ -1,571 +0,0 @@
|
||||
// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/convert.cu
|
||||
// Dequant functions
|
||||
static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int ib, const int iqs, dfloat2 & v){
|
||||
const block_q4_0 * x = (const block_q4_0 *) vx;
|
||||
|
||||
const dfloat d = x[ib].d;
|
||||
|
||||
const int vui = x[ib].qs[iqs];
|
||||
|
||||
v.x = __int2half_rn(vui & 0xF);
|
||||
v.y = __int2half_rn(vui >> 4);
|
||||
|
||||
v = __hsub2(v, __floats2half2_rn(8.0f, 8.0f));
|
||||
v = __hmul2(v, {d, d});
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ void dequantize_q4_1(const void * vx, const int ib, const int iqs, dfloat2 & v){
|
||||
const block_q4_1 * x = (const block_q4_1 *) vx;
|
||||
|
||||
const dfloat d = __low2half(x[ib].dm);
|
||||
const dfloat m = __high2half(x[ib].dm);
|
||||
|
||||
const int vui = x[ib].qs[iqs];
|
||||
|
||||
v.x = __int2half_rn(vui & 0xF);
|
||||
v.y = __int2half_rn(vui >> 4);
|
||||
|
||||
v = __hmul2(v, {d, d});
|
||||
v = __hadd2(v, {m, m});
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ void dequantize_q5_0(const void * vx, const int ib, const int iqs, dfloat2 & v){
|
||||
const block_q5_0 * x = (const block_q5_0 *) vx;
|
||||
|
||||
const dfloat d = x[ib].d;
|
||||
|
||||
uint32_t qh;
|
||||
memcpy(&qh, x[ib].qh, sizeof(qh));
|
||||
|
||||
const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
|
||||
const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
|
||||
|
||||
v.x = __int2half_rn((x[ib].qs[iqs] & 0xf) | xh_0);
|
||||
v.y = __int2half_rn((x[ib].qs[iqs] >> 4) | xh_1);
|
||||
|
||||
v = __hsub2(v, __floats2half2_rn(16.0f, 16.0f));
|
||||
v = __hmul2(v, {d, d});
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ void dequantize_q5_1(const void * vx, const int ib, const int iqs, dfloat2 & v){
|
||||
const block_q5_1 * x = (const block_q5_1 *) vx;
|
||||
|
||||
const dfloat d = __low2half(x[ib].dm);
|
||||
const dfloat m = __high2half(x[ib].dm);
|
||||
|
||||
uint32_t qh;
|
||||
memcpy(&qh, x[ib].qh, sizeof(qh));
|
||||
|
||||
const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
|
||||
const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
|
||||
|
||||
v.x = __int2half_rn((x[ib].qs[iqs] & 0xf) | xh_0);
|
||||
v.y = __int2half_rn((x[ib].qs[iqs] >> 4) | xh_1);
|
||||
|
||||
v = __hmul2(v, {d, d});
|
||||
v = __hadd2(v, {m, m});
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const int ib, const int iqs, dfloat2 & v){
|
||||
const block_q8_0 * x = (const block_q8_0 *) vx;
|
||||
|
||||
const dfloat d = x[ib].d;
|
||||
|
||||
v.x = __int2half_rn(x[ib].qs[iqs + 0]);
|
||||
v.y = __int2half_rn(x[ib].qs[iqs + 1]);
|
||||
|
||||
v = __hmul2(v, {d, d});
|
||||
}
|
||||
|
||||
template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
|
||||
static __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k) {
|
||||
const int64_t i = 2*((int64_t)blockDim.x*blockIdx.x + threadIdx.x);
|
||||
|
||||
if (i >= k) {
|
||||
return;
|
||||
}
|
||||
|
||||
const int ib = i/qk; // block index
|
||||
const int iqs = (i%qk)/qr; // quant index
|
||||
const int iybs = i - i%qk; // y block start index
|
||||
const int y_offset = qr == 1 ? 1 : qk/2;
|
||||
|
||||
// dequantize
|
||||
dfloat2 v;
|
||||
dequantize_kernel(vx, ib, iqs, v);
|
||||
|
||||
y[iybs + iqs + 0] = convert_from_half<dst_t>(v.x);
|
||||
y[iybs + iqs + y_offset] = convert_from_half<dst_t>(v.y);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
|
||||
|
||||
const auto i = blockIdx.x;
|
||||
const block_q2_K * x = (const block_q2_K *) vx;
|
||||
|
||||
const auto tid = threadIdx.x;
|
||||
const int n = tid/32;
|
||||
const int l = tid - 32*n;
|
||||
const int is = 8*n + l/16;
|
||||
|
||||
const uint8_t q = x[i].qs[32*n + l];
|
||||
dst_t * y = yy + i*QK_K + 128*n;
|
||||
|
||||
half dall = __low2half(x[i].dm);
|
||||
half dmin = __high2half(x[i].dm);
|
||||
y[l+ 0] = convert_from_half<dst_t>(__hsub(__hmul(dall, __int2half_rn((x[i].scales[is+0] & 0xF) * ((q >> 0) & 3))), __hmul(dmin, __int2half_rn(x[i].scales[is+0] >> 4))));
|
||||
y[l+32] = convert_from_half<dst_t>(__hsub(__hmul(dall, __int2half_rn((x[i].scales[is+2] & 0xF) * ((q >> 2) & 3))), __hmul(dmin, __int2half_rn(x[i].scales[is+2] >> 4))));
|
||||
y[l+64] = convert_from_half<dst_t>(__hsub(__hmul(dall, __int2half_rn((x[i].scales[is+4] & 0xF) * ((q >> 4) & 3))), __hmul(dmin, __int2half_rn(x[i].scales[is+4] >> 4))));
|
||||
y[l+96] = convert_from_half<dst_t>(__hsub(__hmul(dall, __int2half_rn((x[i].scales[is+6] & 0xF) * ((q >> 6) & 3))), __hmul(dmin, __int2half_rn(x[i].scales[is+6] >> 4))));
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
|
||||
|
||||
const auto i = blockIdx.x;
|
||||
const block_q3_K * x = (const block_q3_K *) vx;
|
||||
|
||||
const auto r = threadIdx.x/4;
|
||||
const int tid = r/2;
|
||||
const int is0 = r%2;
|
||||
const int l0 = 16*is0 + 4*(threadIdx.x%4);
|
||||
const int n = tid / 4;
|
||||
const int j = tid - 4*n;
|
||||
|
||||
uint8_t m = 1 << (4*n + j);
|
||||
int is = 8*n + 2*j + is0;
|
||||
int shift = 2*j;
|
||||
|
||||
int8_t us = is < 4 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+8] >> 0) & 3) << 4) :
|
||||
is < 8 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+4] >> 2) & 3) << 4) :
|
||||
is < 12 ? (x[i].scales[is-8] >> 4) | (((x[i].scales[is+0] >> 4) & 3) << 4) :
|
||||
(x[i].scales[is-8] >> 4) | (((x[i].scales[is-4] >> 6) & 3) << 4);
|
||||
half d_all = x[i].d;
|
||||
half dl = __hmul(d_all, __int2half_rn(us - 32));
|
||||
|
||||
dst_t * y = yy + i*QK_K + 128*n + 32*j;
|
||||
const uint8_t * q = x[i].qs + 32*n;
|
||||
const uint8_t * hm = x[i].hmask;
|
||||
|
||||
for (int l = l0; l < l0+4; ++l) {
|
||||
y[l] = convert_from_half<dst_t>(__hmul(dl, __int2half_rn((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4))));
|
||||
}
|
||||
}
|
||||
|
||||
static inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) {
|
||||
if (j < 4) {
|
||||
d = q[j] & 63; m = q[j + 4] & 63;
|
||||
} else {
|
||||
d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);
|
||||
m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
|
||||
const block_q4_K * x = (const block_q4_K *) vx;
|
||||
|
||||
const auto i = blockIdx.x;
|
||||
|
||||
// assume 32 threads
|
||||
const auto tid = threadIdx.x;
|
||||
const int il = tid/8;
|
||||
const int ir = tid%8;
|
||||
const int is = 2*il;
|
||||
const int n = 4;
|
||||
|
||||
dst_t * y = yy + i*QK_K + 64*il + n*ir;
|
||||
|
||||
const half dall = __low2half(x[i].dm);
|
||||
const half dmin = __high2half(x[i].dm);
|
||||
|
||||
const uint8_t * q = x[i].qs + 32*il + n*ir;
|
||||
|
||||
uint8_t sc, m;
|
||||
get_scale_min_k4(is + 0, x[i].scales, sc, m);
|
||||
const half d1 = __hmul(dall, __int2half_rn(sc));
|
||||
const half m1 = __hmul(dmin, __int2half_rn(m));
|
||||
get_scale_min_k4(is + 1, x[i].scales, sc, m);
|
||||
const half d2 = __hmul(dall, __int2half_rn(sc));
|
||||
const half m2 = __hmul(dmin, __int2half_rn(m));
|
||||
for (int l = 0; l < n; ++l) {
|
||||
y[l + 0] = convert_from_half<dst_t>(__hsub(__hmul(d1, __int2half_rn(q[l] & 0xF)), m1));
|
||||
y[l +32] = convert_from_half<dst_t>(__hsub(__hmul(d2, __int2half_rn(q[l] >> 4)), m2));
|
||||
}
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
|
||||
const block_q5_K * x = (const block_q5_K *) vx;
|
||||
|
||||
const auto i = blockIdx.x;
|
||||
|
||||
// assume 64 threads - this is very slightly better than the one below
|
||||
const auto tid = threadIdx.x;
|
||||
const int il = tid/16; // il is in 0...3
|
||||
const int ir = tid%16; // ir is in 0...15
|
||||
const int is = 2*il; // is is in 0...6
|
||||
|
||||
dst_t * y = yy + i*QK_K + 64*il + 2*ir;
|
||||
|
||||
const half dall = __low2half(x[i].dm);
|
||||
const half dmin = __high2half(x[i].dm);
|
||||
|
||||
const uint8_t * ql = x[i].qs + 32*il + 2*ir;
|
||||
const uint8_t * qh = x[i].qh + 2*ir;
|
||||
|
||||
uint8_t sc, m;
|
||||
get_scale_min_k4(is + 0, x[i].scales, sc, m);
|
||||
const half d1 = __hmul(dall, __int2half_rn(sc)); const half m1 = __hmul(dmin, __int2half_rn(m));
|
||||
get_scale_min_k4(is + 1, x[i].scales, sc, m);
|
||||
const half d2 = __hmul(dall, __int2half_rn(sc)); const half m2 = __hmul(dmin, __int2half_rn(m));
|
||||
|
||||
uint8_t hm = 1 << (2*il);
|
||||
y[ 0] = convert_from_half<dst_t>(__hsub(__hmul(d1, __int2half_rn((ql[0] & 0xF) + (qh[0] & hm ? 16 : 0))), m1));
|
||||
y[ 1] = convert_from_half<dst_t>(__hsub(__hmul(d1, __int2half_rn((ql[1] & 0xF) + (qh[1] & hm ? 16 : 0))), m1));
|
||||
hm <<= 1;
|
||||
y[32] = convert_from_half<dst_t>(__hsub(__hmul(d2, __int2half_rn((ql[0] >> 4) + (qh[0] & hm ? 16 : 0))), m2));
|
||||
y[33] = convert_from_half<dst_t>(__hsub(__hmul(d2, __int2half_rn((ql[1] >> 4) + (qh[1] & hm ? 16 : 0))), m2));
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {
|
||||
const block_q6_K * x = (const block_q6_K *) vx;
|
||||
|
||||
const auto i = blockIdx.x;
|
||||
|
||||
// assume 64 threads - this is very slightly better than the one below
|
||||
const auto tid = threadIdx.x;
|
||||
const int ip = tid/32; // ip is 0 or 1
|
||||
const int il = tid - 32*ip; // 0...32
|
||||
const int is = 8*ip + il/16;
|
||||
|
||||
dst_t * y = yy + i*QK_K + 128*ip + il;
|
||||
|
||||
const half d = x[i].d;
|
||||
|
||||
const uint8_t * ql = x[i].ql + 64*ip + il;
|
||||
const uint8_t qh = x[i].qh[32*ip + il];
|
||||
const int8_t * sc = x[i].scales + is;
|
||||
|
||||
y[ 0] = convert_from_half<dst_t>(__hmul(d, __int2half_rn(sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32))));
|
||||
y[32] = convert_from_half<dst_t>(__hmul(d, __int2half_rn(sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32))));
|
||||
y[64] = convert_from_half<dst_t>(__hmul(d, __int2half_rn(sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32))));
|
||||
y[96] = convert_from_half<dst_t>(__hmul(d, __int2half_rn(sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32))));
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
|
||||
|
||||
const auto i = blockIdx.x;
|
||||
const block_iq2_xxs * x = (const block_iq2_xxs *) vx;
|
||||
|
||||
const auto tid = threadIdx.x;
|
||||
const int il = tid/8; // 0...3
|
||||
const int ib = tid%8; // 0...7
|
||||
dst_t * y = yy + i*QK_K + 32*ib + 8*il;
|
||||
const uint16_t * q2 = x[i].qs + 4*ib;
|
||||
const uint8_t * aux8 = (const uint8_t *)q2;
|
||||
const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]);
|
||||
const uint32_t aux32 = q2[2] | (q2[3] << 16);
|
||||
const float d = __half2float(x[i].d) * (0.5f + (aux32 >> 28)) * 0.25f;
|
||||
const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];
|
||||
for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
|
||||
|
||||
const auto i = blockIdx.x;
|
||||
const block_iq2_xs * x = (const block_iq2_xs *) vx;
|
||||
|
||||
const auto tid = threadIdx.x;
|
||||
const int il = tid/8; // 0...3
|
||||
const int ib = tid%8; // 0...7
|
||||
dst_t * y = yy + i*QK_K + 32*ib + 8*il;
|
||||
const uint16_t * q2 = x[i].qs + 4*ib;
|
||||
const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));
|
||||
const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;
|
||||
const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];
|
||||
for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
|
||||
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
|
||||
|
||||
const auto i = blockIdx.x;
|
||||
const block_iq2_s * x = (const block_iq2_s *) vx;
|
||||
|
||||
const auto tid = threadIdx.x;
|
||||
const int il = tid/8; // 0...3
|
||||
const int ib = tid%8; // 0...7
|
||||
dst_t * y = yy + i*QK_K + 32*ib + 8*il;
|
||||
const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[i].qs[4*ib+il] | ((x[i].qh[ib] << (8-2*il)) & 0x300)));
|
||||
const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;
|
||||
const uint8_t signs = x[i].qs[QK_K/8+4*ib+il];
|
||||
for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
|
||||
|
||||
const auto i = blockIdx.x;
|
||||
const block_iq3_xxs * x = (const block_iq3_xxs *) vx;
|
||||
|
||||
const auto tid = threadIdx.x;
|
||||
const int il = tid/8; // 0...3
|
||||
const int ib = tid%8; // 0...7
|
||||
dst_t * y = yy + i*QK_K + 32*ib + 8*il;
|
||||
const uint8_t * q3 = x[i].qs + 8*ib;
|
||||
const uint16_t * gas = (const uint16_t *)(x[i].qs + QK_K/4) + 2*ib;
|
||||
const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]);
|
||||
const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]);
|
||||
const uint32_t aux32 = gas[0] | (gas[1] << 16);
|
||||
const float d = __half2float(x[i].d) * (0.5f + (aux32 >> 28)) * 0.5f;
|
||||
const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];
|
||||
for (int j = 0; j < 4; ++j) {
|
||||
y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);
|
||||
y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
|
||||
|
||||
const auto i = blockIdx.x;
|
||||
const block_iq3_s * x = (const block_iq3_s *) vx;
|
||||
|
||||
const auto tid = threadIdx.x;
|
||||
const int il = tid/8; // 0...3
|
||||
const int ib = tid%8; // 0...7
|
||||
dst_t * y = yy + i*QK_K + 32*ib + 8*il;
|
||||
const uint8_t * qs = x[i].qs + 8*ib;
|
||||
const uint8_t * grid1 = (const uint8_t *)(iq3xs_grid + (qs[2*il+0] | ((x[i].qh[ib] << (8-2*il)) & 256)));
|
||||
const uint8_t * grid2 = (const uint8_t *)(iq3xs_grid + (qs[2*il+1] | ((x[i].qh[ib] << (7-2*il)) & 256)));
|
||||
const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib/2] >> 4*(ib%2)) & 0xf)) * 0.5f;
|
||||
const uint8_t signs = x[i].signs[4*ib + il];
|
||||
for (int j = 0; j < 4; ++j) {
|
||||
y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);
|
||||
y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {
|
||||
|
||||
const int64_t i = blockIdx.x;
|
||||
const block_iq1_s * x = (const block_iq1_s *) vx;
|
||||
|
||||
const int64_t tid = threadIdx.x;
|
||||
const int64_t il = tid/8; // 0...3
|
||||
const int64_t ib = tid%8; // 0...7
|
||||
dst_t * y = yy + i*QK_K + 32*ib + 8*il;
|
||||
const float delta = x[i].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA;
|
||||
const float d = __half2float(x[i].d) * (2*((x[i].qh[ib] >> 12) & 7) + 1);
|
||||
uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;
|
||||
grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[ib] >> 3*il) & 7) << 8)];
|
||||
grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;
|
||||
grid32[0] &= 0x0f0f0f0f;
|
||||
for (int j = 0; j < 8; ++j) {
|
||||
y[j] = d * (q[j] + delta);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_t * __restrict__ yy) {
|
||||
|
||||
const int64_t i = blockIdx.x;
|
||||
const block_iq1_m * x = (const block_iq1_m *) vx;
|
||||
|
||||
const int64_t tid = threadIdx.x;
|
||||
const int64_t il = tid/8; // 0...3
|
||||
const int64_t ib = tid%8; // 0...7
|
||||
dst_t * y = yy + i*QK_K + 32*ib + 8*il;
|
||||
const uint16_t * sc = (const uint16_t *)x[i].scales;
|
||||
iq1m_scale_t scale;
|
||||
scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
|
||||
const int64_t ib16 = 2*ib + il/2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4);
|
||||
const float d = __half2float(scale.f16) * (2*((sc[ib16/4] >> 3*(ib16%4)) & 0x7) + 1);
|
||||
const float delta = x[i].qh[2*ib+il/2] & (0x08 << 4*(il%2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA;
|
||||
uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;
|
||||
grid32[0] = iq1s_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[2*ib+il/2] >> 4*(il%2)) & 7) << 8)];
|
||||
grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;
|
||||
grid32[0] &= 0x0f0f0f0f;
|
||||
for (int j = 0; j < 8; ++j) {
|
||||
y[j] = d * (q[j] + delta);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) {
|
||||
|
||||
const auto i = blockIdx.x;
|
||||
const block_iq4_nl * x = (const block_iq4_nl *) vx + i*(QK_K/QK4_NL);
|
||||
|
||||
const auto tid = threadIdx.x;
|
||||
const int il = tid/8; // 0...3
|
||||
const int ib = tid%8; // 0...7
|
||||
dst_t * y = yy + i*QK_K + 32*ib + 4*il;
|
||||
const uint8_t * q4 = x[ib].qs + 4*il;
|
||||
const float d = __half2float(x[ib].d);
|
||||
for (int j = 0; j < 4; ++j) {
|
||||
y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];
|
||||
y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {
|
||||
const auto i = blockIdx.x;
|
||||
const block_iq4_xs * x = (const block_iq4_xs *)vx;
|
||||
|
||||
const auto tid = threadIdx.x;
|
||||
const int il = tid/8; // 0...3
|
||||
const int ib = tid%8; // 0...7
|
||||
dst_t * y = yy + i*QK_K + 32*ib + 4*il;
|
||||
const uint8_t * q4 = x[i].qs + 16*ib + 4*il;
|
||||
const float d = __half2float(x[i].d) * ((((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4)) - 32);
|
||||
for (int j = 0; j < 4; ++j) {
|
||||
y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf];
|
||||
y[j+16] = d * kvalues_iq4nl[q4[j] >> 4];
|
||||
}
|
||||
}
|
||||
|
||||
template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
|
||||
static void dequantize_block_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k, cudaStream_t stream) {
|
||||
const int64_t num_blocks = (k + 2*CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2*CUDA_DEQUANTIZE_BLOCK_SIZE);
|
||||
dequantize_block<qk, qr, dequantize_kernel><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static void dequantize_row_q2_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
|
||||
const int nb = k / QK_K;
|
||||
dequantize_block_q2_K<<<nb, 64, 0, stream>>>(vx, y);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
|
||||
const int nb = k / QK_K;
|
||||
dequantize_block_q3_K<<<nb, 64, 0, stream>>>(vx, y);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static void dequantize_row_q4_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
|
||||
const int nb = k / QK_K;
|
||||
dequantize_block_q4_K<<<nb, 32, 0, stream>>>(vx, y);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static void dequantize_row_q5_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
|
||||
const int nb = k / QK_K;
|
||||
dequantize_block_q5_K<<<nb, 64, 0, stream>>>(vx, y);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static void dequantize_row_q6_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
|
||||
const int nb = k / QK_K;
|
||||
dequantize_block_q6_K<<<nb, 64, 0, stream>>>(vx, y);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static void dequantize_row_iq2_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
|
||||
const int nb = k / QK_K;
|
||||
dequantize_block_iq2_xxs<<<nb, 32, 0, stream>>>(vx, y);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
|
||||
const int nb = k / QK_K;
|
||||
dequantize_block_iq2_xs<<<nb, 32, 0, stream>>>(vx, y);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static void dequantize_row_iq2_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
|
||||
const int nb = k / QK_K;
|
||||
dequantize_block_iq2_s<<<nb, 32, 0, stream>>>(vx, y);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
|
||||
const int nb = k / QK_K;
|
||||
dequantize_block_iq3_xxs<<<nb, 32, 0, stream>>>(vx, y);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static void dequantize_row_iq3_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
|
||||
const int nb = k / QK_K;
|
||||
dequantize_block_iq3_s<<<nb, 32, 0, stream>>>(vx, y);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
|
||||
const int nb = k / QK_K;
|
||||
dequantize_block_iq1_s<<<nb, 32, 0, stream>>>(vx, y);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static void dequantize_row_iq1_m_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
|
||||
const int nb = k / QK_K;
|
||||
dequantize_block_iq1_m<<<nb, 32, 0, stream>>>(vx, y);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
|
||||
const int nb = (k + QK_K - 1) / QK_K;
|
||||
dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
|
||||
const int nb = (k + QK_K - 1) / QK_K;
|
||||
dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);
|
||||
}
|
||||
|
||||
template<typename dst_t>
|
||||
static to_cuda_ggml_t<dst_t> ggml_get_to_cuda(int64_t type) {
|
||||
switch (type) {
|
||||
case 2:
|
||||
return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;
|
||||
case 3:
|
||||
return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;
|
||||
case 6:
|
||||
return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;
|
||||
case 7:
|
||||
return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;
|
||||
case 8:
|
||||
return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;
|
||||
case 10:
|
||||
return dequantize_row_q2_K_cuda;
|
||||
case 11:
|
||||
return dequantize_row_q3_K_cuda;
|
||||
case 12:
|
||||
return dequantize_row_q4_K_cuda;
|
||||
case 13:
|
||||
return dequantize_row_q5_K_cuda;
|
||||
case 14:
|
||||
return dequantize_row_q6_K_cuda;
|
||||
case 16:
|
||||
return dequantize_row_iq2_xxs_cuda;
|
||||
case 17:
|
||||
return dequantize_row_iq2_xs_cuda;
|
||||
case 18:
|
||||
return dequantize_row_iq3_xxs_cuda;
|
||||
case 19:
|
||||
return dequantize_row_iq1_s_cuda;
|
||||
case 20:
|
||||
return dequantize_row_iq4_nl_cuda;
|
||||
case 21:
|
||||
return dequantize_row_iq3_s_cuda;
|
||||
case 22:
|
||||
return dequantize_row_iq2_s_cuda;
|
||||
case 23:
|
||||
return dequantize_row_iq4_xs_cuda;
|
||||
case 29:
|
||||
return dequantize_row_iq1_m_cuda;
|
||||
default:
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,561 +0,0 @@
|
||||
#include <cuda_fp16.h>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include "../../../cuda_compat.h"
|
||||
#include "../../dispatch_utils.h"
|
||||
#include "../../torch_utils.h"
|
||||
|
||||
#include <torch/csrc/stable/ops.h>
|
||||
|
||||
#include "ggml-common.h"
|
||||
#include "vecdotq.cuh"
|
||||
#include "dequantize.cuh"
|
||||
#include "mmvq.cuh"
|
||||
#include "mmq.cuh"
|
||||
#include "moe.cuh"
|
||||
#include "moe_vec.cuh"
|
||||
|
||||
// Q8 gemv
|
||||
template <typename scalar_t>
|
||||
static __global__ void quantize_q8_1(const scalar_t* __restrict__ x,
|
||||
void* __restrict__ vy, const int kx,
|
||||
const int kx_padded) {
|
||||
const auto ix = blockDim.x * blockIdx.x + threadIdx.x;
|
||||
if (ix >= kx_padded) {
|
||||
return;
|
||||
}
|
||||
const auto iy = blockDim.y * blockIdx.y + threadIdx.y;
|
||||
const int i_padded = iy * kx_padded + ix;
|
||||
|
||||
block_q8_1* y = (block_q8_1*)vy;
|
||||
|
||||
const int ib = i_padded / QK8_1; // block index
|
||||
const int iqs = i_padded % QK8_1; // quant index
|
||||
|
||||
const float xi = ix < kx ? static_cast<float>(x[iy * kx + ix]) : 0.0f;
|
||||
float amax = fabsf(xi);
|
||||
float sum = xi;
|
||||
|
||||
#pragma unroll
|
||||
for (int mask = 16; mask > 0; mask >>= 1) {
|
||||
amax = fmaxf(amax, VLLM_SHFL_XOR_SYNC_WIDTH(amax, mask, 32));
|
||||
sum += VLLM_SHFL_XOR_SYNC_WIDTH(sum, mask, 32);
|
||||
}
|
||||
|
||||
const float d = amax / 127;
|
||||
const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);
|
||||
|
||||
y[ib].qs[iqs] = q;
|
||||
|
||||
if (iqs > 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
y[ib].ds.x = __float2half(d);
|
||||
y[ib].ds.y = __float2half(sum);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void quantize_row_q8_1_cuda(const scalar_t* x, void* vy, const int kx,
|
||||
const int ky, cudaStream_t stream) {
|
||||
const int64_t kx_padded = (kx + 512 - 1) / 512 * 512;
|
||||
const int block_num_x =
|
||||
(kx_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;
|
||||
constexpr int MAX_BLOCK_SIZE = 65535;
|
||||
for (int off = 0; off < ky; off += MAX_BLOCK_SIZE) {
|
||||
const int num_blocks_y = std::min(ky, off + MAX_BLOCK_SIZE) - off;
|
||||
const dim3 num_blocks(block_num_x, num_blocks_y, 1);
|
||||
const dim3 block_size(CUDA_DEQUANTIZE_BLOCK_SIZE, 1, 1);
|
||||
quantize_q8_1<<<num_blocks, block_size, 0, stream>>>(
|
||||
&x[off * kx], (int32_t*)vy + off * (kx_padded / 32 * 9), kx, kx_padded);
|
||||
}
|
||||
}
|
||||
|
||||
torch::stable::Tensor ggml_dequantize(
|
||||
torch::stable::Tensor W, // quant weight
|
||||
int64_t type, int64_t m, int64_t n,
|
||||
std::optional<torch::headeronly::ScalarType> const& dtype) {
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
W.get_device_index());
|
||||
auto dtype_ = dtype.value_or(torch::headeronly::ScalarType::Half);
|
||||
auto DW = torch::stable::empty({m, n}, dtype_, std::nullopt, W.device());
|
||||
torch::stable::fill_(DW, 0.0);
|
||||
cudaStream_t stream = get_current_cuda_stream();
|
||||
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(DW.scalar_type(), "ggml_dequantize", [&] {
|
||||
auto to_cuda = ggml_get_to_cuda<scalar_t>(type);
|
||||
to_cuda((void*)W.data_ptr(), (scalar_t*)DW.data_ptr(), m * n, stream);
|
||||
});
|
||||
|
||||
return DW;
|
||||
}
|
||||
|
||||
torch::stable::Tensor ggml_mul_mat_vec_a8(
|
||||
torch::stable::Tensor W, // quant weight
|
||||
torch::stable::Tensor X, // input
|
||||
int64_t type, int64_t row) {
|
||||
int64_t col = X.sizes()[1];
|
||||
int64_t vecs = X.sizes()[0];
|
||||
const int64_t padded = (col + 512 - 1) / 512 * 512;
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
X.get_device_index());
|
||||
auto Y = torch::stable::empty({vecs, row}, X.scalar_type(), std::nullopt,
|
||||
W.device());
|
||||
torch::stable::fill_(Y, 0.0);
|
||||
cudaStream_t stream = get_current_cuda_stream();
|
||||
auto quant_X = torch::stable::empty({vecs, padded / 32 * 9},
|
||||
torch::headeronly::ScalarType::Int,
|
||||
std::nullopt, W.device());
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
X.scalar_type(), "ggml_mul_mat_vec_a8", [&] {
|
||||
quantize_row_q8_1_cuda<scalar_t>((scalar_t*)X.data_ptr(),
|
||||
(void*)quant_X.data_ptr(), col, vecs,
|
||||
stream);
|
||||
switch (type) {
|
||||
case 2:
|
||||
mul_mat_vec_q4_0_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 3:
|
||||
mul_mat_vec_q4_1_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 6:
|
||||
mul_mat_vec_q5_0_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 7:
|
||||
mul_mat_vec_q5_1_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 8:
|
||||
mul_mat_vec_q8_0_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 10:
|
||||
mul_mat_vec_q2_K_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 11:
|
||||
mul_mat_vec_q3_K_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 12:
|
||||
mul_mat_vec_q4_K_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 13:
|
||||
mul_mat_vec_q5_K_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 14:
|
||||
mul_mat_vec_q6_K_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 16:
|
||||
mul_mat_vec_iq2_xxs_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 17:
|
||||
mul_mat_vec_iq2_xs_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 18:
|
||||
mul_mat_vec_iq3_xxs_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 19:
|
||||
mul_mat_vec_iq1_s_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 20:
|
||||
mul_mat_vec_iq4_nl_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 21:
|
||||
mul_mat_vec_iq3_s_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 22:
|
||||
mul_mat_vec_iq2_s_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 23:
|
||||
mul_mat_vec_iq4_xs_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
case 29:
|
||||
mul_mat_vec_iq1_m_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, vecs, stream);
|
||||
break;
|
||||
}
|
||||
});
|
||||
return Y;
|
||||
}
|
||||
|
||||
torch::stable::Tensor ggml_mul_mat_a8(torch::stable::Tensor W, // quant weight
|
||||
torch::stable::Tensor X, // input
|
||||
int64_t type, int64_t row) {
|
||||
int64_t col = X.sizes()[1];
|
||||
int64_t padded = (col + 512 - 1) / 512 * 512;
|
||||
int64_t batch = X.sizes()[0];
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
X.get_device_index());
|
||||
auto Y = torch::stable::empty({batch, row}, X.scalar_type(), std::nullopt,
|
||||
W.device());
|
||||
torch::stable::fill_(Y, 0.0);
|
||||
cudaStream_t stream = get_current_cuda_stream();
|
||||
auto quant_X = torch::stable::empty({batch, padded / 32 * 9},
|
||||
torch::headeronly::ScalarType::Int,
|
||||
std::nullopt, W.device());
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(X.scalar_type(), "ggml_mul_mat_a8", [&] {
|
||||
quantize_row_q8_1_cuda((scalar_t*)X.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
col, batch, stream);
|
||||
|
||||
switch (type) {
|
||||
case 2:
|
||||
ggml_mul_mat_q4_0_q8_1_cuda(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
|
||||
break;
|
||||
case 3:
|
||||
ggml_mul_mat_q4_1_q8_1_cuda(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
|
||||
break;
|
||||
case 6:
|
||||
ggml_mul_mat_q5_0_q8_1_cuda(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
|
||||
break;
|
||||
case 7:
|
||||
ggml_mul_mat_q5_1_q8_1_cuda(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
|
||||
break;
|
||||
case 8:
|
||||
ggml_mul_mat_q8_0_q8_1_cuda(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
|
||||
break;
|
||||
case 10:
|
||||
ggml_mul_mat_q2_K_q8_1_cuda(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
|
||||
break;
|
||||
case 11:
|
||||
ggml_mul_mat_q3_K_q8_1_cuda(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
|
||||
break;
|
||||
case 12:
|
||||
ggml_mul_mat_q4_K_q8_1_cuda(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
|
||||
break;
|
||||
case 13:
|
||||
ggml_mul_mat_q5_K_q8_1_cuda(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
|
||||
break;
|
||||
case 14:
|
||||
ggml_mul_mat_q6_K_q8_1_cuda(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), col, row, batch, padded, row, stream);
|
||||
break;
|
||||
}
|
||||
});
|
||||
return Y;
|
||||
}
|
||||
|
||||
torch::stable::Tensor ggml_moe_a8(torch::stable::Tensor X, // input
|
||||
torch::stable::Tensor W, // expert weights
|
||||
torch::stable::Tensor sorted_token_ids,
|
||||
torch::stable::Tensor expert_ids,
|
||||
torch::stable::Tensor num_tokens_post_padded,
|
||||
int64_t type, int64_t row, int64_t top_k,
|
||||
int64_t tokens) {
|
||||
int64_t col = X.sizes()[1];
|
||||
int64_t padded = (col + 512 - 1) / 512 * 512;
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
X.get_device_index());
|
||||
auto Y = torch::stable::empty({tokens * top_k, row}, X.scalar_type(),
|
||||
std::nullopt, W.device());
|
||||
torch::stable::fill_(Y, 0.0);
|
||||
cudaStream_t stream = get_current_cuda_stream();
|
||||
auto quant_X = torch::stable::empty({tokens, padded / 32 * 9},
|
||||
torch::headeronly::ScalarType::Int,
|
||||
std::nullopt, W.device());
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(X.scalar_type(), "ggml_moe_a8", [&] {
|
||||
quantize_row_q8_1_cuda((scalar_t*)X.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
col, tokens, stream);
|
||||
switch (type) {
|
||||
case 2:
|
||||
ggml_moe_q4_0_q8_1_cuda(
|
||||
(void*)quant_X.data_ptr(), (void*)W.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
|
||||
(int*)expert_ids.data_ptr(),
|
||||
(int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
|
||||
tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
|
||||
break;
|
||||
case 3:
|
||||
ggml_moe_q4_1_q8_1_cuda(
|
||||
(void*)quant_X.data_ptr(), (void*)W.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
|
||||
(int*)expert_ids.data_ptr(),
|
||||
(int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
|
||||
tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
|
||||
break;
|
||||
case 6:
|
||||
ggml_moe_q5_0_q8_1_cuda(
|
||||
(void*)quant_X.data_ptr(), (void*)W.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
|
||||
(int*)expert_ids.data_ptr(),
|
||||
(int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
|
||||
tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
|
||||
break;
|
||||
case 7:
|
||||
ggml_moe_q5_1_q8_1_cuda(
|
||||
(void*)quant_X.data_ptr(), (void*)W.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
|
||||
(int*)expert_ids.data_ptr(),
|
||||
(int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
|
||||
tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
|
||||
break;
|
||||
case 8:
|
||||
ggml_moe_q8_0_q8_1_cuda(
|
||||
(void*)quant_X.data_ptr(), (void*)W.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
|
||||
(int*)expert_ids.data_ptr(),
|
||||
(int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
|
||||
tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
|
||||
break;
|
||||
case 10:
|
||||
ggml_moe_q2_K_q8_1_cuda(
|
||||
(void*)quant_X.data_ptr(), (void*)W.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
|
||||
(int*)expert_ids.data_ptr(),
|
||||
(int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
|
||||
tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
|
||||
break;
|
||||
case 11:
|
||||
ggml_moe_q3_K_q8_1_cuda(
|
||||
(void*)quant_X.data_ptr(), (void*)W.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
|
||||
(int*)expert_ids.data_ptr(),
|
||||
(int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
|
||||
tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
|
||||
break;
|
||||
case 12:
|
||||
ggml_moe_q4_K_q8_1_cuda(
|
||||
(void*)quant_X.data_ptr(), (void*)W.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
|
||||
(int*)expert_ids.data_ptr(),
|
||||
(int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
|
||||
tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
|
||||
break;
|
||||
case 13:
|
||||
ggml_moe_q5_K_q8_1_cuda(
|
||||
(void*)quant_X.data_ptr(), (void*)W.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
|
||||
(int*)expert_ids.data_ptr(),
|
||||
(int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
|
||||
tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
|
||||
break;
|
||||
case 14:
|
||||
ggml_moe_q6_K_q8_1_cuda(
|
||||
(void*)quant_X.data_ptr(), (void*)W.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)sorted_token_ids.data_ptr(),
|
||||
(int*)expert_ids.data_ptr(),
|
||||
(int*)num_tokens_post_padded.data_ptr(), W.stride(0), col, row,
|
||||
tokens, padded, row, top_k, sorted_token_ids.sizes()[0], stream);
|
||||
break;
|
||||
}
|
||||
});
|
||||
return Y;
|
||||
}
|
||||
|
||||
torch::stable::Tensor ggml_moe_a8_vec(
|
||||
torch::stable::Tensor X, // input
|
||||
torch::stable::Tensor W, // expert weights
|
||||
torch::stable::Tensor topk_ids, int64_t top_k, int64_t type, int64_t row,
|
||||
int64_t tokens) {
|
||||
int64_t col = X.sizes()[1];
|
||||
const int64_t padded = (col + 512 - 1) / 512 * 512;
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
X.get_device_index());
|
||||
auto Y = torch::stable::empty({tokens * top_k, row}, X.scalar_type(),
|
||||
std::nullopt, W.device());
|
||||
torch::stable::fill_(Y, 0.0);
|
||||
cudaStream_t stream = get_current_cuda_stream();
|
||||
auto quant_X = torch::stable::empty({tokens, padded / 32 * 9},
|
||||
torch::headeronly::ScalarType::Int,
|
||||
std::nullopt, W.device());
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(X.scalar_type(), "ggml_moe_vec_a8", [&] {
|
||||
quantize_row_q8_1_cuda<scalar_t>((scalar_t*)X.data_ptr(),
|
||||
(void*)quant_X.data_ptr(), col, tokens,
|
||||
stream);
|
||||
switch (type) {
|
||||
case 2:
|
||||
moe_vec_q4_0_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 3:
|
||||
moe_vec_q4_1_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 6:
|
||||
moe_vec_q5_0_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 7:
|
||||
moe_vec_q5_1_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 8:
|
||||
moe_vec_q8_0_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 10:
|
||||
moe_vec_q2_K_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 11:
|
||||
moe_vec_q3_K_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 12:
|
||||
moe_vec_q4_K_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 13:
|
||||
moe_vec_q5_K_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 14:
|
||||
moe_vec_q6_K_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 16:
|
||||
moe_vec_iq2_xxs_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 17:
|
||||
moe_vec_iq2_xs_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 18:
|
||||
moe_vec_iq3_xxs_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 19:
|
||||
moe_vec_iq1_s_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 20:
|
||||
moe_vec_iq4_nl_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 21:
|
||||
moe_vec_iq3_s_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 22:
|
||||
moe_vec_iq2_s_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 23:
|
||||
moe_vec_iq4_xs_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
case 29:
|
||||
moe_vec_iq1_m_q8_1_cuda<scalar_t>(
|
||||
(void*)W.data_ptr(), (void*)quant_X.data_ptr(),
|
||||
(scalar_t*)Y.data_ptr(), (int*)topk_ids.data_ptr(), top_k, tokens,
|
||||
col, row, quant_X.stride(0), stream);
|
||||
break;
|
||||
}
|
||||
});
|
||||
return Y;
|
||||
}
|
||||
|
||||
int64_t ggml_moe_get_block_size(int64_t type) {
|
||||
switch (type) {
|
||||
case 2:
|
||||
return MOE_X_Q4_0;
|
||||
case 3:
|
||||
return MOE_X_Q4_1;
|
||||
case 6:
|
||||
return MOE_X_Q5_0;
|
||||
case 7:
|
||||
return MOE_X_Q5_1;
|
||||
case 8:
|
||||
return MOE_X_Q8_0;
|
||||
case 10:
|
||||
return MOE_X_Q2_K;
|
||||
case 11:
|
||||
return MOE_X_Q3_K;
|
||||
case 12:
|
||||
return MOE_X_Q4_K;
|
||||
case 13:
|
||||
return MOE_X_Q5_K;
|
||||
case 14:
|
||||
return MOE_X_Q6_K;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -1,610 +0,0 @@
|
||||
// copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmq.cu
|
||||
template <typename scalar_t, int qk, int qr, int qi, bool need_sum, typename block_q_t, int mmq_x, int mmq_y, int nwarps,
|
||||
allocate_tiles_cuda_t allocate_tiles, load_tiles_cuda_t load_tiles, int vdr, vec_dot_q_mul_mat_cuda_t vec_dot>
|
||||
static __device__ __forceinline__ void mul_mat_q(
|
||||
const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
|
||||
const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
|
||||
|
||||
const block_q_t * x = (const block_q_t *) vx;
|
||||
const block_q8_1 * y = (const block_q8_1 *) vy;
|
||||
|
||||
const int blocks_per_row_x = ncols_x / qk;
|
||||
const int blocks_per_col_y = nrows_y / QK8_1;
|
||||
const int blocks_per_warp = WARP_SIZE_GGUF / qi;
|
||||
|
||||
const int & ncols_dst = ncols_y;
|
||||
|
||||
const auto row_dst_0 = blockIdx.x*mmq_y;
|
||||
const int & row_x_0 = row_dst_0;
|
||||
|
||||
const auto col_dst_0 = blockIdx.y*mmq_x;
|
||||
const int & col_y_0 = col_dst_0;
|
||||
|
||||
int * tile_x_ql = nullptr;
|
||||
half2 * tile_x_dm = nullptr;
|
||||
int * tile_x_qh = nullptr;
|
||||
int * tile_x_sc = nullptr;
|
||||
|
||||
allocate_tiles(&tile_x_ql, &tile_x_dm, &tile_x_qh, &tile_x_sc);
|
||||
|
||||
__shared__ int tile_y_qs[mmq_x * WARP_SIZE_GGUF];
|
||||
__shared__ half2 tile_y_ds[mmq_x * WARP_SIZE_GGUF/QI8_1];
|
||||
|
||||
float sum[mmq_y/WARP_SIZE_GGUF][mmq_x/nwarps] = {{0.0f}};
|
||||
|
||||
for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) {
|
||||
|
||||
load_tiles(x + row_x_0*blocks_per_row_x + ib0, tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc,
|
||||
threadIdx.y, nrows_x-row_x_0-1, threadIdx.x, blocks_per_row_x);
|
||||
|
||||
#pragma unroll
|
||||
for (int ir = 0; ir < qr && ib0 + ir * blocks_per_warp/qr < blocks_per_row_x; ++ir) {
|
||||
const auto kqs = ir*WARP_SIZE_GGUF + threadIdx.x;
|
||||
const int kbxd = kqs / QI8_1;
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < mmq_x; i += nwarps) {
|
||||
const int col_y_eff = min(col_y_0 + threadIdx.y + i, ncols_y-1); // to prevent out-of-bounds memory accesses
|
||||
const block_q8_1 * by0 = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + kbxd];
|
||||
const int index_y = (threadIdx.y + i) * WARP_SIZE_GGUF + kqs % WARP_SIZE_GGUF;
|
||||
tile_y_qs[index_y] = get_int_from_int8_aligned(by0->qs, threadIdx.x % QI8_1);
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int ids0 = 0; ids0 < mmq_x; ids0 += nwarps * QI8_1) {
|
||||
const int ids = (ids0 + threadIdx.y * QI8_1 + threadIdx.x / (WARP_SIZE_GGUF/QI8_1)) % mmq_x;
|
||||
const auto kby = threadIdx.x % (WARP_SIZE_GGUF/QI8_1);
|
||||
const int col_y_eff = min(col_y_0 + ids, ncols_y-1);
|
||||
|
||||
// if the sum is not needed it's faster to transform the scale to f32 ahead of time
|
||||
const half2 * dsi_src = &y[col_y_eff*blocks_per_col_y + ib0 * (qk/QK8_1) + ir*(WARP_SIZE_GGUF/QI8_1) + kby].ds;
|
||||
half2 * dsi_dst = &tile_y_ds[ids * (WARP_SIZE_GGUF/QI8_1) + kby];
|
||||
if (need_sum) {
|
||||
*dsi_dst = *dsi_src;
|
||||
} else {
|
||||
float * dfi_dst = (float *) dsi_dst;
|
||||
*dfi_dst = __low2float(*dsi_src);
|
||||
}
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
// #pragma unroll // unrolling this loop causes too much register pressure
|
||||
for (int k = ir*WARP_SIZE_GGUF/qr; k < (ir+1)*WARP_SIZE_GGUF/qr; k += vdr) {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < mmq_x; j += nwarps) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
|
||||
sum[i/WARP_SIZE_GGUF][j/nwarps] += vec_dot(
|
||||
tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc, tile_y_qs, tile_y_ds,
|
||||
threadIdx.x + i, threadIdx.y + j, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int j = 0; j < mmq_x; j += nwarps) {
|
||||
const auto col_dst = col_dst_0 + j + threadIdx.y;
|
||||
if (col_dst >= ncols_dst) {
|
||||
return;
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
|
||||
const auto row_dst = row_dst_0 + threadIdx.x + i;
|
||||
if (row_dst >= nrows_dst) {
|
||||
continue;
|
||||
}
|
||||
dst[col_dst*nrows_dst + row_dst] = sum[i/WARP_SIZE_GGUF][j/nwarps];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MMQ_X_Q4_0 64
|
||||
#define MMQ_Y_Q4_0 128
|
||||
#define NWARPS_Q4_0 8
|
||||
#else
|
||||
#define MMQ_X_Q4_0 4
|
||||
#define MMQ_Y_Q4_0 32
|
||||
#define NWARPS_Q4_0 4
|
||||
#endif
|
||||
|
||||
template<typename scalar_t, bool need_check> static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q4_0, 2)
|
||||
#endif
|
||||
mul_mat_q4_0(
|
||||
const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
|
||||
const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
|
||||
const int mmq_x = MMQ_X_Q4_0;
|
||||
const int mmq_y = MMQ_Y_Q4_0;
|
||||
const int nwarps = NWARPS_Q4_0;
|
||||
|
||||
mul_mat_q<scalar_t, QK4_0, QR4_0, QI4_0, true, block_q4_0, mmq_x, mmq_y, nwarps, allocate_tiles_q4_0<mmq_y>,
|
||||
load_tiles_q4_0<mmq_y, nwarps, need_check>, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void ggml_mul_mat_q4_0_q8_1_cuda(
|
||||
const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
|
||||
|
||||
int mmq_x = MMQ_X_Q4_0;
|
||||
int mmq_y = MMQ_Y_Q4_0;
|
||||
int nwarps = NWARPS_Q4_0;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
const bool need_check = false;
|
||||
mul_mat_q4_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
} else {
|
||||
const bool need_check = true;
|
||||
mul_mat_q4_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MMQ_X_Q4_1 64
|
||||
#define MMQ_Y_Q4_1 128
|
||||
#define NWARPS_Q4_1 8
|
||||
#else
|
||||
#define MMQ_X_Q4_1 4
|
||||
#define MMQ_Y_Q4_1 32
|
||||
#define NWARPS_Q4_1 4
|
||||
#endif
|
||||
|
||||
template<typename scalar_t, bool need_check> static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q4_1, 2)
|
||||
#endif
|
||||
mul_mat_q4_1(
|
||||
const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
|
||||
const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
|
||||
const int mmq_x = MMQ_X_Q4_1;
|
||||
const int mmq_y = MMQ_Y_Q4_1;
|
||||
const int nwarps = NWARPS_Q4_1;
|
||||
|
||||
mul_mat_q<scalar_t, QK4_1, QR4_1, QI4_1, true, block_q4_1, mmq_x, mmq_y, nwarps, allocate_tiles_q4_1<mmq_y>,
|
||||
load_tiles_q4_1<mmq_y, nwarps, need_check>, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void ggml_mul_mat_q4_1_q8_1_cuda(
|
||||
const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
|
||||
|
||||
int mmq_x = MMQ_X_Q4_1;
|
||||
int mmq_y = MMQ_Y_Q4_1;
|
||||
int nwarps = NWARPS_Q4_1;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
const bool need_check = false;
|
||||
mul_mat_q4_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
} else {
|
||||
const bool need_check = true;
|
||||
mul_mat_q4_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MMQ_X_Q5_0 64
|
||||
#define MMQ_Y_Q5_0 128
|
||||
#define NWARPS_Q5_0 8
|
||||
#else
|
||||
#define MMQ_X_Q5_0 4
|
||||
#define MMQ_Y_Q5_0 32
|
||||
#define NWARPS_Q5_0 4
|
||||
#endif
|
||||
|
||||
template<typename scalar_t, bool need_check> static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q5_0, 2)
|
||||
#endif
|
||||
mul_mat_q5_0(
|
||||
const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
|
||||
const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
|
||||
const int mmq_x = MMQ_X_Q5_0;
|
||||
const int mmq_y = MMQ_Y_Q5_0;
|
||||
const int nwarps = NWARPS_Q5_0;
|
||||
|
||||
mul_mat_q<scalar_t, QK5_0, QR5_0, QI5_0, false, block_q5_0, mmq_x, mmq_y, nwarps, allocate_tiles_q5_0<mmq_y>,
|
||||
load_tiles_q5_0<mmq_y, nwarps, need_check>, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void ggml_mul_mat_q5_0_q8_1_cuda(
|
||||
const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
|
||||
|
||||
const int mmq_x = MMQ_X_Q5_0;
|
||||
const int mmq_y = MMQ_Y_Q5_0;
|
||||
const int nwarps = NWARPS_Q5_0;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
const bool need_check = false;
|
||||
mul_mat_q5_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
} else {
|
||||
const bool need_check = true;
|
||||
mul_mat_q5_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MMQ_X_Q5_1 64
|
||||
#define MMQ_Y_Q5_1 128
|
||||
#define NWARPS_Q5_1 8
|
||||
#else
|
||||
#define MMQ_X_Q5_1 4
|
||||
#define MMQ_Y_Q5_1 32
|
||||
#define NWARPS_Q5_1 4
|
||||
#endif
|
||||
|
||||
template<typename scalar_t, bool need_check> static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q5_1, 2)
|
||||
#endif
|
||||
mul_mat_q5_1(
|
||||
const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
|
||||
const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
|
||||
const int mmq_x = MMQ_X_Q5_1;
|
||||
const int mmq_y = MMQ_Y_Q5_1;
|
||||
const int nwarps = NWARPS_Q5_1;
|
||||
|
||||
mul_mat_q<scalar_t, QK5_1, QR5_1, QI5_1, true, block_q5_1, mmq_x, mmq_y, nwarps, allocate_tiles_q5_1<mmq_y>,
|
||||
load_tiles_q5_1<mmq_y, nwarps, need_check>, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void ggml_mul_mat_q5_1_q8_1_cuda(
|
||||
const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
|
||||
const int mmq_x = MMQ_X_Q5_1;
|
||||
const int mmq_y = MMQ_Y_Q5_1;
|
||||
const int nwarps = NWARPS_Q5_1;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
const bool need_check = false;
|
||||
mul_mat_q5_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
} else {
|
||||
const bool need_check = true;
|
||||
mul_mat_q5_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MMQ_X_Q8_0 64
|
||||
#define MMQ_Y_Q8_0 128
|
||||
#define NWARPS_Q8_0 8
|
||||
#else
|
||||
#define MMQ_X_Q8_0 4
|
||||
#define MMQ_Y_Q8_0 32
|
||||
#define NWARPS_Q8_0 4
|
||||
#endif
|
||||
|
||||
template<typename scalar_t, bool need_check> static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q8_0, 2)
|
||||
#endif
|
||||
mul_mat_q8_0(
|
||||
const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
|
||||
const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
|
||||
const int mmq_x = MMQ_X_Q8_0;
|
||||
const int mmq_y = MMQ_Y_Q8_0;
|
||||
const int nwarps = NWARPS_Q8_0;
|
||||
|
||||
mul_mat_q<scalar_t, QK8_0, QR8_0, QI8_0, false, block_q8_0, mmq_x, mmq_y, nwarps, allocate_tiles_q8_0<mmq_y>,
|
||||
load_tiles_q8_0<mmq_y, nwarps, need_check>, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void ggml_mul_mat_q8_0_q8_1_cuda(
|
||||
const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
|
||||
const int mmq_x = MMQ_X_Q8_0;
|
||||
const int mmq_y = MMQ_Y_Q8_0;
|
||||
const int nwarps = NWARPS_Q8_0;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
const bool need_check = false;
|
||||
mul_mat_q8_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
} else {
|
||||
const bool need_check = true;
|
||||
mul_mat_q8_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MMQ_X_Q2_K 64
|
||||
#define MMQ_Y_Q2_K 128
|
||||
#define NWARPS_Q2_K 8
|
||||
#else
|
||||
#define MMQ_X_Q2_K 4
|
||||
#define MMQ_Y_Q2_K 32
|
||||
#define NWARPS_Q2_K 4
|
||||
#endif
|
||||
|
||||
template<typename scalar_t, bool need_check> static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q2_K, 2)
|
||||
#endif
|
||||
mul_mat_q2_K(
|
||||
const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
|
||||
const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
|
||||
const int mmq_x = MMQ_X_Q2_K;
|
||||
const int mmq_y = MMQ_Y_Q2_K;
|
||||
const int nwarps = NWARPS_Q2_K;
|
||||
|
||||
mul_mat_q<scalar_t, QK_K, QR2_K, QI2_K, false, block_q2_K, mmq_x, mmq_y, nwarps, allocate_tiles_q2_K<mmq_y>,
|
||||
load_tiles_q2_K<mmq_y, nwarps, need_check>, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void ggml_mul_mat_q2_K_q8_1_cuda(
|
||||
const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
|
||||
const int mmq_x = MMQ_X_Q2_K;
|
||||
const int mmq_y = MMQ_Y_Q2_K;
|
||||
const int nwarps = NWARPS_Q2_K;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
const bool need_check = false;
|
||||
mul_mat_q2_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
} else {
|
||||
const bool need_check = true;
|
||||
mul_mat_q2_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MMQ_X_Q3_K 64
|
||||
#define MMQ_Y_Q3_K 128
|
||||
#define NWARPS_Q3_K 8
|
||||
#else
|
||||
#define MMQ_X_Q3_K 4
|
||||
#define MMQ_Y_Q3_K 32
|
||||
#define NWARPS_Q3_K 4
|
||||
#endif
|
||||
|
||||
template<typename scalar_t, bool need_check> static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q3_K, 2)
|
||||
#endif
|
||||
mul_mat_q3_K(
|
||||
const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
|
||||
const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
|
||||
|
||||
const int mmq_x = MMQ_X_Q3_K;
|
||||
const int mmq_y = MMQ_Y_Q3_K;
|
||||
const int nwarps = NWARPS_Q3_K;
|
||||
|
||||
mul_mat_q<scalar_t, QK_K, QR3_K, QI3_K, false, block_q3_K, mmq_x, mmq_y, nwarps, allocate_tiles_q3_K<mmq_y>,
|
||||
load_tiles_q3_K<mmq_y, nwarps, need_check>, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void ggml_mul_mat_q3_K_q8_1_cuda(
|
||||
const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
|
||||
|
||||
const int mmq_x = MMQ_X_Q3_K;
|
||||
const int mmq_y = MMQ_Y_Q3_K;
|
||||
const int nwarps = NWARPS_Q3_K;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
const bool need_check = false;
|
||||
mul_mat_q3_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
} else {
|
||||
const bool need_check = true;
|
||||
mul_mat_q3_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MMQ_X_Q4_K 64
|
||||
#define MMQ_Y_Q4_K 128
|
||||
#define NWARPS_Q4_K 8
|
||||
#else
|
||||
#define MMQ_X_Q4_K 4
|
||||
#define MMQ_Y_Q4_K 32
|
||||
#define NWARPS_Q4_K 4
|
||||
#endif
|
||||
|
||||
template<typename scalar_t, bool need_check> static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q4_K, 2)
|
||||
#endif
|
||||
mul_mat_q4_K(
|
||||
const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
|
||||
const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
|
||||
const int mmq_x = MMQ_X_Q4_K;
|
||||
const int mmq_y = MMQ_Y_Q4_K;
|
||||
const int nwarps = NWARPS_Q4_K;
|
||||
|
||||
mul_mat_q<scalar_t, QK_K, QR4_K, QI4_K, true, block_q4_K, mmq_x, mmq_y, nwarps, allocate_tiles_q4_K<mmq_y>,
|
||||
load_tiles_q4_K<mmq_y, nwarps, need_check>, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void ggml_mul_mat_q4_K_q8_1_cuda(
|
||||
const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
|
||||
const int mmq_x = MMQ_X_Q4_K;
|
||||
const int mmq_y = MMQ_Y_Q4_K;
|
||||
const int nwarps = NWARPS_Q4_K;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
const bool need_check = false;
|
||||
mul_mat_q4_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
} else {
|
||||
const bool need_check = true;
|
||||
mul_mat_q4_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MMQ_X_Q5_K 64
|
||||
#define MMQ_Y_Q5_K 128
|
||||
#define NWARPS_Q5_K 8
|
||||
#else
|
||||
#define MMQ_X_Q5_K 4
|
||||
#define MMQ_Y_Q5_K 32
|
||||
#define NWARPS_Q5_K 4
|
||||
#endif
|
||||
|
||||
template<typename scalar_t, bool need_check> static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q5_K, 2)
|
||||
#endif
|
||||
mul_mat_q5_K(
|
||||
const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
|
||||
const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
|
||||
const int mmq_x = MMQ_X_Q5_K;
|
||||
const int mmq_y = MMQ_Y_Q5_K;
|
||||
const int nwarps = NWARPS_Q5_K;
|
||||
|
||||
mul_mat_q<scalar_t, QK_K, QR5_K, QI5_K, true, block_q5_K, mmq_x, mmq_y, nwarps, allocate_tiles_q5_K<mmq_y>,
|
||||
load_tiles_q5_K<mmq_y, nwarps, need_check>, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void ggml_mul_mat_q5_K_q8_1_cuda(
|
||||
const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
|
||||
|
||||
const int mmq_x = MMQ_X_Q5_K;
|
||||
const int mmq_y = MMQ_Y_Q5_K;
|
||||
const int nwarps = NWARPS_Q5_K;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
const bool need_check = false;
|
||||
mul_mat_q5_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
} else {
|
||||
const bool need_check = true;
|
||||
mul_mat_q5_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MMQ_X_Q6_K 64
|
||||
#define MMQ_Y_Q6_K 128
|
||||
#define NWARPS_Q6_K 8
|
||||
#else
|
||||
#define MMQ_X_Q6_K 4
|
||||
#define MMQ_Y_Q6_K 32
|
||||
#define NWARPS_Q6_K 4
|
||||
#endif
|
||||
|
||||
template<typename scalar_t, bool need_check> static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF*NWARPS_Q6_K, 2)
|
||||
#endif
|
||||
mul_mat_q6_K(
|
||||
const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst,
|
||||
const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) {
|
||||
const int mmq_x = MMQ_X_Q6_K;
|
||||
const int mmq_y = MMQ_Y_Q6_K;
|
||||
const int nwarps = NWARPS_Q6_K;
|
||||
|
||||
mul_mat_q<scalar_t, QK_K, QR6_K, QI6_K, false, block_q6_K, mmq_x, mmq_y, nwarps, allocate_tiles_q6_K<mmq_y>,
|
||||
load_tiles_q6_K<mmq_y, nwarps, need_check>, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void ggml_mul_mat_q6_K_q8_1_cuda(
|
||||
const void * vx, const void * vy, scalar_t * dst, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) {
|
||||
const int mmq_x = MMQ_X_Q6_K;
|
||||
const int mmq_y = MMQ_Y_Q6_K;
|
||||
const int nwarps = NWARPS_Q6_K;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
const bool need_check = false;
|
||||
mul_mat_q6_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
} else {
|
||||
const bool need_check = true;
|
||||
mul_mat_q6_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>
|
||||
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
|
||||
}
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmvq.cu
|
||||
template <typename scalar_t, int qk, int qi, typename block_q_t, int vdr, vec_dot_q_cuda_t vec_dot_q_cuda>
|
||||
static __global__ void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict__ vy, scalar_t * __restrict__ dst, const int ncols, const int nrows, const int nvecs) {
|
||||
const auto row = blockIdx.x*blockDim.y + threadIdx.y;
|
||||
const auto vec = blockIdx.y;
|
||||
|
||||
if (row >= nrows || vec >= nvecs) {
|
||||
return;
|
||||
}
|
||||
|
||||
const int blocks_per_row = ncols / qk;
|
||||
const int blocks_per_warp = vdr * WARP_SIZE / qi;
|
||||
const int nrows_y = (ncols + 512 - 1) / 512 * 512;
|
||||
|
||||
|
||||
// partial sum for each thread
|
||||
float tmp = 0.0f;
|
||||
|
||||
const block_q_t * x = (const block_q_t *) vx;
|
||||
const block_q8_1 * y = (const block_q8_1 *) vy;
|
||||
|
||||
for (auto i = threadIdx.x / (qi/vdr); i < blocks_per_row; i += blocks_per_warp) {
|
||||
const int ibx = row*blocks_per_row + i; // x block index
|
||||
|
||||
const int iby = vec*(nrows_y/QK8_1) + i * (qk/QK8_1); // y block index that aligns with ibx
|
||||
|
||||
const int iqs = vdr * (threadIdx.x % (qi/vdr)); // x block quant index when casting the quants to int
|
||||
|
||||
tmp += vec_dot_q_cuda(&x[ibx], &y[iby], iqs);
|
||||
}
|
||||
|
||||
// sum up partial sums and write back result
|
||||
#pragma unroll
|
||||
for (int mask = WARP_SIZE/2; mask > 0; mask >>= 1) {
|
||||
tmp += VLLM_SHFL_XOR_SYNC(tmp, mask);
|
||||
}
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
dst[vec*nrows + row] = tmp;
|
||||
}
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_q4_0_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_q4_1_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK4_0, QI4_1, block_q4_1, VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_q5_0_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK5_0, QI5_0, block_q5_0, VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_q5_1_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK5_1, QI5_1, block_q5_1, VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_q8_0_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK8_0, QI8_0, block_q8_0, VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_q2_K_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK_K, QI2_K, block_q2_K, VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_q3_K_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK_K, QI3_K, block_q3_K, VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_q4_K_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK_K, QI4_K, block_q4_K, VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_q5_K_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK_K, QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_q6_K_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK_K, QI6_K, block_q6_K, VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_iq2_xxs_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK_K, QI2_XXS, block_iq2_xxs, 1, vec_dot_iq2_xxs_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_iq2_xs_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK_K, QI2_XS, block_iq2_xs, 1, vec_dot_iq2_xs_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_iq2_s_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK_K, QI2_S, block_iq2_s, 1, vec_dot_iq2_s_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_iq3_xxs_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK_K, QI3_XXS, block_iq3_xxs, 1, vec_dot_iq3_xxs_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_iq1_s_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK_K, QI1_S, block_iq1_s, 1, vec_dot_iq1_s_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_iq1_m_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK_K, QI1_M, block_iq1_m, 1, vec_dot_iq1_m_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_iq4_nl_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK4_NL, QI4_NL, block_iq4_nl, VDR_Q4_0_Q8_1_MMVQ, vec_dot_iq4_nl_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_iq4_xs_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK_K, QI4_XS, block_iq4_xs, 1, vec_dot_iq4_xs_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
|
||||
template<typename scalar_t>
|
||||
static void mul_mat_vec_iq3_s_q8_1_cuda(const void * vx, const void * vy, scalar_t * dst, const int ncols, const int nrows, const int nvecs, cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, nvecs, 1);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
mul_mat_vec_q<scalar_t, QK_K, QI3_XS, block_iq3_s, 1, vec_dot_iq3_s_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);
|
||||
}
|
||||
@@ -1,739 +0,0 @@
|
||||
#include <cstdint>
|
||||
|
||||
/* Adapted from ./csrc/quantization/gguf/mmq.cuh
|
||||
based on ./vllm/model_executor/layers/fused_moe/experts/triton_moe.py */
|
||||
template <typename scalar_t, int qk, int qr, int qi, bool need_sum,
|
||||
typename block_q_t, int mmq_x, int mmq_y, int nwarps,
|
||||
allocate_tiles_cuda_t allocate_tiles, load_tiles_cuda_t load_tiles,
|
||||
int vdr, vec_dot_q_mul_mat_cuda_t vec_dot>
|
||||
static __device__ __forceinline__ void moe_q(
|
||||
const void* __restrict__ vx, const void* __restrict__ vy,
|
||||
scalar_t* __restrict__ dst, const int* __restrict__ sorted_token_ids,
|
||||
const int* __restrict__ expert_ids,
|
||||
const int* __restrict__ num_tokens_post_padded, const int exp_stride,
|
||||
const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y,
|
||||
const int nrows_dst, const int top_k) {
|
||||
const int blocks_per_row_x = ncols_x / qk;
|
||||
const int blocks_per_col_y = nrows_y / QK8_1;
|
||||
const int blocks_per_warp = WARP_SIZE_GGUF / qi;
|
||||
|
||||
const int ncols_dst = ncols_y * top_k;
|
||||
|
||||
const auto row_dst_0 = blockIdx.x * mmq_y;
|
||||
const int& row_x_0 = row_dst_0;
|
||||
|
||||
const auto col_dst_0 = blockIdx.y * mmq_x;
|
||||
|
||||
int token_offs[mmq_x / nwarps];
|
||||
for (int i = 0; i < mmq_x; i += nwarps) {
|
||||
token_offs[i / nwarps] = sorted_token_ids[col_dst_0 + threadIdx.y + i];
|
||||
}
|
||||
|
||||
const int exp_idx = expert_ids[blockIdx.y];
|
||||
if (exp_idx > 255 || exp_idx < 0) return;
|
||||
if (blockIdx.y * mmq_x > num_tokens_post_padded[0]) return;
|
||||
|
||||
const block_q_t* x = (const block_q_t*)((char*)vx + exp_idx * exp_stride);
|
||||
const block_q8_1* y = (const block_q8_1*)(vy);
|
||||
|
||||
int* tile_x_ql = nullptr;
|
||||
half2* tile_x_dm = nullptr;
|
||||
int* tile_x_qh = nullptr;
|
||||
int* tile_x_sc = nullptr;
|
||||
|
||||
allocate_tiles(&tile_x_ql, &tile_x_dm, &tile_x_qh, &tile_x_sc);
|
||||
|
||||
__shared__ int tile_y_qs[mmq_x * WARP_SIZE_GGUF];
|
||||
__shared__ half2 tile_y_ds[mmq_x * WARP_SIZE_GGUF / QI8_1];
|
||||
|
||||
float sum[mmq_y / WARP_SIZE_GGUF][mmq_x / nwarps] = {{0.0f}};
|
||||
|
||||
for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) {
|
||||
load_tiles(x + row_x_0 * blocks_per_row_x + ib0, tile_x_ql, tile_x_dm,
|
||||
tile_x_qh, tile_x_sc, threadIdx.y, nrows_x - row_x_0 - 1,
|
||||
threadIdx.x, blocks_per_row_x);
|
||||
|
||||
const int n_per_r = ((qk * blocks_per_warp) / qr);
|
||||
#pragma unroll
|
||||
for (int ir = 0; ir < qr && ib0 * qk + ir * n_per_r < ncols_x; ++ir) {
|
||||
const auto kqs = ir * WARP_SIZE_GGUF + threadIdx.x;
|
||||
const int kbxd = kqs / QI8_1;
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < mmq_x; i += nwarps) {
|
||||
const int col_y_eff = token_offs[i / nwarps] / top_k;
|
||||
const int block_x = ib0 * (qk / QK8_1) + kbxd;
|
||||
if (col_y_eff < ncols_y && block_x < blocks_per_col_y) {
|
||||
const block_q8_1* by0 = &y[col_y_eff * blocks_per_col_y + block_x];
|
||||
const int index_y =
|
||||
(threadIdx.y + i) * WARP_SIZE_GGUF + kqs % WARP_SIZE_GGUF;
|
||||
tile_y_qs[index_y] =
|
||||
get_int_from_int8_aligned(by0->qs, threadIdx.x % QI8_1);
|
||||
}
|
||||
}
|
||||
|
||||
if (threadIdx.x < n_per_r / QK8_1) {
|
||||
const auto kby = threadIdx.x % (WARP_SIZE_GGUF / QI8_1);
|
||||
const int col_y_eff = token_offs[threadIdx.y] / top_k;
|
||||
const int block_x =
|
||||
ib0 * (qk / QK8_1) + ir * (WARP_SIZE_GGUF / QI8_1) + kby;
|
||||
|
||||
if (col_y_eff < ncols_y && block_x < blocks_per_col_y) {
|
||||
const half2* dsi_src = &y[col_y_eff * blocks_per_col_y + block_x].ds;
|
||||
half2* dsi_dst =
|
||||
&tile_y_ds[threadIdx.y * (WARP_SIZE_GGUF / QI8_1) + kby];
|
||||
|
||||
if (need_sum) {
|
||||
*dsi_dst = *dsi_src;
|
||||
} else {
|
||||
float* dfi_dst = (float*)dsi_dst;
|
||||
*dfi_dst = __low2float(*dsi_src);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// #pragma unroll // unrolling this loop causes too much register pressure
|
||||
for (int k = ir * WARP_SIZE_GGUF / qr; k < (ir + 1) * WARP_SIZE_GGUF / qr;
|
||||
k += vdr) {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < mmq_x; j += nwarps) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
|
||||
sum[i / WARP_SIZE_GGUF][j / nwarps] +=
|
||||
vec_dot(tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc, tile_y_qs,
|
||||
tile_y_ds, threadIdx.x + i, threadIdx.y + j, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int j = 0; j < mmq_x; j += nwarps) {
|
||||
const int col_dst = token_offs[j / nwarps];
|
||||
if (col_dst >= ncols_dst) {
|
||||
return;
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
|
||||
const auto row_dst = row_dst_0 + threadIdx.x + i;
|
||||
if (row_dst >= nrows_dst) {
|
||||
continue;
|
||||
}
|
||||
dst[col_dst * nrows_dst + row_dst] = sum[i / WARP_SIZE_GGUF][j / nwarps];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MOE_X_Q4_0 8
|
||||
#define MOE_Y_Q4_0 128
|
||||
#define NWARPS_Q4_0 8
|
||||
#else
|
||||
#define MOE_X_Q4_0 4
|
||||
#define MOE_Y_Q4_0 32
|
||||
#define NWARPS_Q4_0 4
|
||||
#endif
|
||||
|
||||
template <typename scalar_t, bool need_check>
|
||||
static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_0, 2)
|
||||
#endif
|
||||
moe_q4_0(const void* __restrict__ vx, const void* __restrict__ vy,
|
||||
scalar_t* __restrict__ dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst,
|
||||
const int top_k) {
|
||||
const int mmq_x = MOE_X_Q4_0;
|
||||
const int mmq_y = MOE_Y_Q4_0;
|
||||
const int nwarps = NWARPS_Q4_0;
|
||||
|
||||
moe_q<scalar_t, QK4_0, QR4_0, QI4_0, true, block_q4_0, mmq_x, mmq_y, nwarps,
|
||||
allocate_tiles_q4_0<mmq_y>, load_tiles_q4_0<mmq_y, nwarps, need_check>,
|
||||
VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat>(
|
||||
vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void ggml_moe_q4_0_q8_1_cuda(
|
||||
const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
|
||||
const int tokens_post_padded, cudaStream_t stream) {
|
||||
int mmq_x = MOE_X_Q4_0;
|
||||
int mmq_y = MOE_Y_Q4_0;
|
||||
int nwarps = NWARPS_Q4_0;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (tokens_post_padded) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
constexpr bool need_check = false;
|
||||
moe_q4_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
} else {
|
||||
constexpr bool need_check = true;
|
||||
moe_q4_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MOE_X_Q4_1 8
|
||||
#define MOE_Y_Q4_1 128
|
||||
#define NWARPS_Q4_1 8
|
||||
#else
|
||||
#define MOE_X_Q4_1 4
|
||||
#define MOE_Y_Q4_1 32
|
||||
#define NWARPS_Q4_1 4
|
||||
#endif
|
||||
|
||||
template <typename scalar_t, bool need_check>
|
||||
static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_1, 2)
|
||||
#endif
|
||||
moe_q4_1(const void* __restrict__ vx, const void* __restrict__ vy,
|
||||
scalar_t* __restrict__ dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst,
|
||||
const int top_k) {
|
||||
const int mmq_x = MOE_X_Q4_1;
|
||||
const int mmq_y = MOE_Y_Q4_1;
|
||||
const int nwarps = NWARPS_Q4_1;
|
||||
|
||||
moe_q<scalar_t, QK4_1, QR4_1, QI4_1, true, block_q4_1, mmq_x, mmq_y, nwarps,
|
||||
allocate_tiles_q4_1<mmq_y>, load_tiles_q4_1<mmq_y, nwarps, need_check>,
|
||||
VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat>(
|
||||
vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void ggml_moe_q4_1_q8_1_cuda(
|
||||
const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
|
||||
const int tokens_post_padded, cudaStream_t stream) {
|
||||
int mmq_x = MOE_X_Q4_1;
|
||||
int mmq_y = MOE_Y_Q4_1;
|
||||
int nwarps = NWARPS_Q4_1;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (tokens_post_padded) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
constexpr bool need_check = false;
|
||||
moe_q4_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
} else {
|
||||
constexpr bool need_check = true;
|
||||
moe_q4_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MOE_X_Q5_0 8
|
||||
#define MOE_Y_Q5_0 128
|
||||
#define NWARPS_Q5_0 8
|
||||
#else
|
||||
#define MOE_X_Q5_0 4
|
||||
#define MOE_Y_Q5_0 32
|
||||
#define NWARPS_Q5_0 4
|
||||
#endif
|
||||
|
||||
template <typename scalar_t, bool need_check>
|
||||
static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_0, 2)
|
||||
#endif
|
||||
moe_q5_0(const void* __restrict__ vx, const void* __restrict__ vy,
|
||||
scalar_t* __restrict__ dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst,
|
||||
const int top_k) {
|
||||
const int mmq_x = MOE_X_Q5_0;
|
||||
const int mmq_y = MOE_Y_Q5_0;
|
||||
const int nwarps = NWARPS_Q5_0;
|
||||
|
||||
moe_q<scalar_t, QK5_0, QR5_0, QI5_0, false, block_q5_0, mmq_x, mmq_y, nwarps,
|
||||
allocate_tiles_q5_0<mmq_y>, load_tiles_q5_0<mmq_y, nwarps, need_check>,
|
||||
VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat>(
|
||||
vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void ggml_moe_q5_0_q8_1_cuda(
|
||||
const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
|
||||
const int tokens_post_padded, cudaStream_t stream) {
|
||||
const int mmq_x = MOE_X_Q5_0;
|
||||
const int mmq_y = MOE_Y_Q5_0;
|
||||
const int nwarps = NWARPS_Q5_0;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (tokens_post_padded) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
constexpr bool need_check = false;
|
||||
moe_q5_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
} else {
|
||||
constexpr bool need_check = true;
|
||||
moe_q5_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MOE_X_Q5_1 8
|
||||
#define MOE_Y_Q5_1 128
|
||||
#define NWARPS_Q5_1 8
|
||||
#else
|
||||
#define MOE_X_Q5_1 4
|
||||
#define MOE_Y_Q5_1 32
|
||||
#define NWARPS_Q5_1 4
|
||||
#endif
|
||||
|
||||
template <typename scalar_t, bool need_check>
|
||||
static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_1, 2)
|
||||
#endif
|
||||
moe_q5_1(const void* __restrict__ vx, const void* __restrict__ vy,
|
||||
scalar_t* __restrict__ dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst,
|
||||
const int top_k) {
|
||||
const int mmq_x = MOE_X_Q5_1;
|
||||
const int mmq_y = MOE_Y_Q5_1;
|
||||
const int nwarps = NWARPS_Q5_1;
|
||||
|
||||
moe_q<scalar_t, QK5_1, QR5_1, QI5_1, true, block_q5_1, mmq_x, mmq_y, nwarps,
|
||||
allocate_tiles_q5_1<mmq_y>, load_tiles_q5_1<mmq_y, nwarps, need_check>,
|
||||
VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat>(
|
||||
vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void ggml_moe_q5_1_q8_1_cuda(
|
||||
const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
|
||||
const int tokens_post_padded, cudaStream_t stream) {
|
||||
const int mmq_x = MOE_X_Q5_1;
|
||||
const int mmq_y = MOE_Y_Q5_1;
|
||||
const int nwarps = NWARPS_Q5_1;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (tokens_post_padded) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
constexpr bool need_check = false;
|
||||
moe_q5_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
} else {
|
||||
constexpr bool need_check = true;
|
||||
moe_q5_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MOE_X_Q8_0 8
|
||||
#define MOE_Y_Q8_0 128
|
||||
#define NWARPS_Q8_0 8
|
||||
#else
|
||||
#define MOE_X_Q8_0 4
|
||||
#define MOE_Y_Q8_0 32
|
||||
#define NWARPS_Q8_0 4
|
||||
#endif
|
||||
|
||||
template <typename scalar_t, bool need_check>
|
||||
static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q8_0, 2)
|
||||
#endif
|
||||
moe_q8_0(const void* __restrict__ vx, const void* __restrict__ vy,
|
||||
scalar_t* __restrict__ dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst,
|
||||
const int top_k) {
|
||||
const int mmq_x = MOE_X_Q8_0;
|
||||
const int mmq_y = MOE_Y_Q8_0;
|
||||
const int nwarps = NWARPS_Q8_0;
|
||||
|
||||
moe_q<scalar_t, QK8_0, QR8_0, QI8_0, false, block_q8_0, mmq_x, mmq_y, nwarps,
|
||||
allocate_tiles_q8_0<mmq_y>, load_tiles_q8_0<mmq_y, nwarps, need_check>,
|
||||
VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat>(
|
||||
vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void ggml_moe_q8_0_q8_1_cuda(
|
||||
const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
|
||||
const int tokens_post_padded, cudaStream_t stream) {
|
||||
const int mmq_x = MOE_X_Q8_0;
|
||||
const int mmq_y = MOE_Y_Q8_0;
|
||||
const int nwarps = NWARPS_Q8_0;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (tokens_post_padded) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
constexpr bool need_check = false;
|
||||
moe_q8_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
} else {
|
||||
constexpr bool need_check = true;
|
||||
moe_q8_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MOE_X_Q2_K 8
|
||||
#define MOE_Y_Q2_K 128
|
||||
#define NWARPS_Q2_K 8
|
||||
#else
|
||||
#define MOE_X_Q2_K 4
|
||||
#define MOE_Y_Q2_K 32
|
||||
#define NWARPS_Q2_K 4
|
||||
#endif
|
||||
|
||||
template <typename scalar_t, bool need_check>
|
||||
static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q2_K, 2)
|
||||
#endif
|
||||
moe_q2_K(const void* __restrict__ vx, const void* __restrict__ vy,
|
||||
scalar_t* __restrict__ dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst,
|
||||
const int top_k) {
|
||||
const int mmq_x = MOE_X_Q2_K;
|
||||
const int mmq_y = MOE_Y_Q2_K;
|
||||
const int nwarps = NWARPS_Q2_K;
|
||||
|
||||
moe_q<scalar_t, QK_K, QR2_K, QI2_K, false, block_q2_K, mmq_x, mmq_y, nwarps,
|
||||
allocate_tiles_q2_K<mmq_y>, load_tiles_q2_K<mmq_y, nwarps, need_check>,
|
||||
VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat>(
|
||||
vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void ggml_moe_q2_K_q8_1_cuda(
|
||||
const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
|
||||
const int tokens_post_padded, cudaStream_t stream) {
|
||||
const int mmq_x = MOE_X_Q2_K;
|
||||
const int mmq_y = MOE_Y_Q2_K;
|
||||
const int nwarps = NWARPS_Q2_K;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (tokens_post_padded) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
constexpr bool need_check = false;
|
||||
moe_q2_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
} else {
|
||||
constexpr bool need_check = true;
|
||||
moe_q2_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MOE_X_Q3_K 8
|
||||
#define MOE_Y_Q3_K 128
|
||||
#define NWARPS_Q3_K 8
|
||||
#else
|
||||
#define MOE_X_Q3_K 4
|
||||
#define MOE_Y_Q3_K 32
|
||||
#define NWARPS_Q3_K 4
|
||||
#endif
|
||||
|
||||
template <typename scalar_t, bool need_check>
|
||||
static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q3_K, 2)
|
||||
#endif
|
||||
moe_q3_K(const void* __restrict__ vx, const void* __restrict__ vy,
|
||||
scalar_t* __restrict__ dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst,
|
||||
const int top_k) {
|
||||
|
||||
const int mmq_x = MOE_X_Q3_K;
|
||||
const int mmq_y = MOE_Y_Q3_K;
|
||||
const int nwarps = NWARPS_Q3_K;
|
||||
|
||||
moe_q<scalar_t, QK_K, QR3_K, QI3_K, false, block_q3_K, mmq_x, mmq_y, nwarps,
|
||||
allocate_tiles_q3_K<mmq_y>, load_tiles_q3_K<mmq_y, nwarps, need_check>,
|
||||
VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat>(
|
||||
vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
template <typename scalar_t>
|
||||
static void ggml_moe_q3_K_q8_1_cuda(
|
||||
const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
|
||||
const int tokens_post_padded, cudaStream_t stream) {
|
||||
const int mmq_x = MOE_X_Q3_K;
|
||||
const int mmq_y = MOE_Y_Q3_K;
|
||||
const int nwarps = NWARPS_Q3_K;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (tokens_post_padded) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
constexpr bool need_check = false;
|
||||
moe_q3_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
} else {
|
||||
constexpr bool need_check = true;
|
||||
moe_q3_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MOE_X_Q4_K 8
|
||||
#define MOE_Y_Q4_K 128
|
||||
#define NWARPS_Q4_K 8
|
||||
#else
|
||||
#define MOE_X_Q4_K 4
|
||||
#define MOE_Y_Q4_K 32
|
||||
#define NWARPS_Q4_K 4
|
||||
#endif
|
||||
|
||||
template <typename scalar_t, bool need_check>
|
||||
static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_K, 2)
|
||||
#endif
|
||||
moe_q4_K(const void* __restrict__ vx, const void* __restrict__ vy,
|
||||
scalar_t* __restrict__ dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst,
|
||||
const int top_k) {
|
||||
const int mmq_x = MOE_X_Q4_K;
|
||||
const int mmq_y = MOE_Y_Q4_K;
|
||||
const int nwarps = NWARPS_Q4_K;
|
||||
|
||||
moe_q<scalar_t, QK_K, QR4_K, QI4_K, true, block_q4_K, mmq_x, mmq_y, nwarps,
|
||||
allocate_tiles_q4_K<mmq_y>, load_tiles_q4_K<mmq_y, nwarps, need_check>,
|
||||
VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat>(
|
||||
vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void ggml_moe_q4_K_q8_1_cuda(
|
||||
const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
|
||||
const int tokens_post_padded, cudaStream_t stream) {
|
||||
const int mmq_x = MOE_X_Q4_K;
|
||||
const int mmq_y = MOE_Y_Q4_K;
|
||||
const int nwarps = NWARPS_Q4_K;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (tokens_post_padded) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
constexpr bool need_check = false;
|
||||
moe_q4_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
} else {
|
||||
constexpr bool need_check = true;
|
||||
moe_q4_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MOE_X_Q5_K 8
|
||||
#define MOE_Y_Q5_K 128
|
||||
#define NWARPS_Q5_K 8
|
||||
#else
|
||||
#define MOE_X_Q5_K 4
|
||||
#define MOE_Y_Q5_K 32
|
||||
#define NWARPS_Q5_K 4
|
||||
#endif
|
||||
|
||||
template <typename scalar_t, bool need_check>
|
||||
static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_K, 2)
|
||||
#endif
|
||||
moe_q5_K(const void* __restrict__ vx, const void* __restrict__ vy,
|
||||
scalar_t* __restrict__ dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst,
|
||||
const int top_k) {
|
||||
const int mmq_x = MOE_X_Q5_K;
|
||||
const int mmq_y = MOE_Y_Q5_K;
|
||||
const int nwarps = NWARPS_Q5_K;
|
||||
|
||||
moe_q<scalar_t, QK_K, QR5_K, QI5_K, true, block_q5_K, mmq_x, mmq_y, nwarps,
|
||||
allocate_tiles_q5_K<mmq_y>, load_tiles_q5_K<mmq_y, nwarps, need_check>,
|
||||
VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat>(
|
||||
vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void ggml_moe_q5_K_q8_1_cuda(
|
||||
const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
|
||||
const int tokens_post_padded, cudaStream_t stream) {
|
||||
const int mmq_x = MOE_X_Q5_K;
|
||||
const int mmq_y = MOE_Y_Q5_K;
|
||||
const int nwarps = NWARPS_Q5_K;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (tokens_post_padded) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
constexpr bool need_check = false;
|
||||
moe_q5_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
} else {
|
||||
constexpr bool need_check = true;
|
||||
moe_q5_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
#define MOE_X_Q6_K 8
|
||||
#define MOE_Y_Q6_K 128
|
||||
#define NWARPS_Q6_K 8
|
||||
#else
|
||||
#define MOE_X_Q6_K 4
|
||||
#define MOE_Y_Q6_K 32
|
||||
#define NWARPS_Q6_K 4
|
||||
#endif
|
||||
|
||||
template <typename scalar_t, bool need_check>
|
||||
static __global__ void
|
||||
#if defined(USE_ROCM)
|
||||
__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q6_K, 2)
|
||||
#endif
|
||||
moe_q6_K(const void* __restrict__ vx, const void* __restrict__ vy,
|
||||
scalar_t* __restrict__ dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst,
|
||||
const int top_k) {
|
||||
const int mmq_x = MOE_X_Q6_K;
|
||||
const int mmq_y = MOE_Y_Q6_K;
|
||||
const int nwarps = NWARPS_Q6_K;
|
||||
|
||||
moe_q<scalar_t, QK_K, QR6_K, QI6_K, false, block_q6_K, mmq_x, mmq_y, nwarps,
|
||||
allocate_tiles_q6_K<mmq_y>, load_tiles_q6_K<mmq_y, nwarps, need_check>,
|
||||
VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat>(
|
||||
vx, vy, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void ggml_moe_q6_K_q8_1_cuda(
|
||||
const void* inp, const void* w, scalar_t* dst, const int* sorted_token_ids,
|
||||
const int* expert_ids, const int* num_tokens_post_padded,
|
||||
const int exp_stride, const int ncols_x, const int nrows_x,
|
||||
const int ncols_y, const int nrows_y, const int nrows_dst, const int top_k,
|
||||
const int tokens_post_padded, cudaStream_t stream) {
|
||||
const int mmq_x = MOE_X_Q6_K;
|
||||
const int mmq_y = MOE_Y_Q6_K;
|
||||
const int nwarps = NWARPS_Q6_K;
|
||||
|
||||
const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;
|
||||
const int block_num_y = (tokens_post_padded) / mmq_x;
|
||||
const dim3 block_nums(block_num_x, block_num_y, 1);
|
||||
const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);
|
||||
|
||||
if (nrows_x % mmq_y == 0) {
|
||||
constexpr bool need_check = false;
|
||||
moe_q6_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
} else {
|
||||
constexpr bool need_check = true;
|
||||
moe_q6_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(
|
||||
w, inp, dst, sorted_token_ids, expert_ids, num_tokens_post_padded,
|
||||
exp_stride, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst, top_k);
|
||||
}
|
||||
}
|
||||
@@ -1,338 +0,0 @@
|
||||
// copied and adapted from
|
||||
// https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmvq.cu
|
||||
template <typename scalar_t, int qk, int qi, typename block_q_t, int vdr,
|
||||
vec_dot_q_cuda_t vec_dot_q_cuda>
|
||||
static __global__ void moe_vec_q(const void* __restrict__ vx,
|
||||
const void* __restrict__ vy,
|
||||
scalar_t* __restrict__ dst,
|
||||
const int* topk_ids, const int topk,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride) {
|
||||
const auto row = blockIdx.x * blockDim.y + threadIdx.y;
|
||||
|
||||
const auto token = blockIdx.z / topk;
|
||||
const auto expert = (topk_ids)[blockIdx.z];
|
||||
|
||||
if (row >= nrows) {
|
||||
return;
|
||||
}
|
||||
|
||||
const int blocks_per_row = ncols / qk;
|
||||
const int blocks_per_warp = vdr * WARP_SIZE / qi;
|
||||
|
||||
// partial sum for each thread
|
||||
float tmp = 0.0f;
|
||||
|
||||
const block_q_t* x = ((const block_q_t*)vx) + expert * nrows * blocks_per_row;
|
||||
const block_q8_1* y =
|
||||
(const block_q8_1*)(((const int*)vy) + token * token_stride);
|
||||
|
||||
for (auto i = threadIdx.x / (qi / vdr); i < blocks_per_row;
|
||||
i += blocks_per_warp) {
|
||||
const int ibx = row * blocks_per_row + i; // x block index
|
||||
|
||||
const int iby = i * (qk / QK8_1); // y block index that aligns with ibx
|
||||
|
||||
const int iqs =
|
||||
vdr *
|
||||
(threadIdx.x %
|
||||
(qi / vdr)); // x block quant index when casting the quants to int
|
||||
|
||||
tmp += vec_dot_q_cuda(&x[ibx], &y[iby], iqs);
|
||||
}
|
||||
|
||||
// sum up partial sums and write back result
|
||||
#pragma unroll
|
||||
for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
|
||||
tmp += VLLM_SHFL_XOR_SYNC(tmp, mask);
|
||||
}
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
dst[blockIdx.z * nrows + row] = tmp;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_q4_0_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ,
|
||||
vec_dot_q4_0_q8_1><<<block_nums, block_dims, 0, stream>>>(
|
||||
vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_q4_1_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK4_0, QI4_1, block_q4_1, VDR_Q4_1_Q8_1_MMVQ,
|
||||
vec_dot_q4_1_q8_1><<<block_nums, block_dims, 0, stream>>>(
|
||||
vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_q5_0_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK5_0, QI5_0, block_q5_0, VDR_Q5_0_Q8_1_MMVQ,
|
||||
vec_dot_q5_0_q8_1><<<block_nums, block_dims, 0, stream>>>(
|
||||
vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_q5_1_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK5_1, QI5_1, block_q5_1, VDR_Q5_1_Q8_1_MMVQ,
|
||||
vec_dot_q5_1_q8_1><<<block_nums, block_dims, 0, stream>>>(
|
||||
vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_q8_0_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK8_0, QI8_0, block_q8_0, VDR_Q8_0_Q8_1_MMVQ,
|
||||
vec_dot_q8_0_q8_1><<<block_nums, block_dims, 0, stream>>>(
|
||||
vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_q2_K_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK_K, QI2_K, block_q2_K, VDR_Q2_K_Q8_1_MMVQ,
|
||||
vec_dot_q2_K_q8_1><<<block_nums, block_dims, 0, stream>>>(
|
||||
vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_q3_K_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK_K, QI3_K, block_q3_K, VDR_Q3_K_Q8_1_MMVQ,
|
||||
vec_dot_q3_K_q8_1><<<block_nums, block_dims, 0, stream>>>(
|
||||
vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_q4_K_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK_K, QI4_K, block_q4_K, VDR_Q4_K_Q8_1_MMVQ,
|
||||
vec_dot_q4_K_q8_1><<<block_nums, block_dims, 0, stream>>>(
|
||||
vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_q5_K_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK_K, QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ,
|
||||
vec_dot_q5_K_q8_1><<<block_nums, block_dims, 0, stream>>>(
|
||||
vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_q6_K_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK_K, QI6_K, block_q6_K, VDR_Q6_K_Q8_1_MMVQ,
|
||||
vec_dot_q6_K_q8_1><<<block_nums, block_dims, 0, stream>>>(
|
||||
vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_iq2_xxs_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK_K, QI2_XXS, block_iq2_xxs, 1, vec_dot_iq2_xxs_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k,
|
||||
ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_iq2_xs_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK_K, QI2_XS, block_iq2_xs, 1, vec_dot_iq2_xs_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k,
|
||||
ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_iq2_s_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK_K, QI2_S, block_iq2_s, 1, vec_dot_iq2_s_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k,
|
||||
ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_iq3_xxs_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK_K, QI3_XXS, block_iq3_xxs, 1, vec_dot_iq3_xxs_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k,
|
||||
ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_iq1_s_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK_K, QI1_S, block_iq1_s, 1, vec_dot_iq1_s_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k,
|
||||
ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_iq1_m_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK_K, QI1_M, block_iq1_m, 1, vec_dot_iq1_m_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k,
|
||||
ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_iq4_nl_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK4_NL, QI4_NL, block_iq4_nl, VDR_Q4_0_Q8_1_MMVQ,
|
||||
vec_dot_iq4_nl_q8_1><<<block_nums, block_dims, 0, stream>>>(
|
||||
vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_iq4_xs_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK_K, QI4_XS, block_iq4_xs, 1, vec_dot_iq4_xs_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k,
|
||||
ncols, nrows, token_stride);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
static void moe_vec_iq3_s_q8_1_cuda(const void* vx, const void* vy,
|
||||
scalar_t* dst, const int* topk_ids,
|
||||
const int top_k, const int tokens,
|
||||
const int ncols, const int nrows,
|
||||
const int token_stride,
|
||||
cudaStream_t stream) {
|
||||
const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;
|
||||
const dim3 block_nums(block_num_y, 1, tokens * top_k);
|
||||
const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);
|
||||
moe_vec_q<scalar_t, QK_K, QI3_XS, block_iq3_s, 1, vec_dot_iq3_s_q8_1>
|
||||
<<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k,
|
||||
ncols, nrows, token_stride);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -557,34 +557,6 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
// Post processing for GPTQ.
|
||||
ops.def("gptq_shuffle(Tensor! q_weight, Tensor q_perm, int bit) -> ()");
|
||||
|
||||
// Dequantization for GGML.
|
||||
ops.def(
|
||||
"ggml_dequantize(Tensor W, int type, SymInt m, SymInt n, ScalarType? "
|
||||
"dtype) -> Tensor");
|
||||
|
||||
// mmvq kernel for GGML.
|
||||
ops.def(
|
||||
"ggml_mul_mat_vec_a8(Tensor W, Tensor X, int type, SymInt row) "
|
||||
"-> Tensor");
|
||||
|
||||
// mmq kernel for GGML.
|
||||
ops.def(
|
||||
"ggml_mul_mat_a8(Tensor W, Tensor X, int type, SymInt row) -> Tensor");
|
||||
|
||||
// moe kernel for GGML.
|
||||
ops.def(
|
||||
"ggml_moe_a8(Tensor X, Tensor W, "
|
||||
"Tensor sorted_token_ids, Tensor expert_ids, Tensor "
|
||||
"num_tokens_post_padded, "
|
||||
"int type, SymInt row, SymInt top_k, SymInt tokens) -> Tensor");
|
||||
|
||||
ops.def(
|
||||
"ggml_moe_a8_vec(Tensor X, Tensor W, "
|
||||
"Tensor topk_ids, int top_k, "
|
||||
"int type, SymInt row, SymInt tokens) -> Tensor");
|
||||
|
||||
ops.def("ggml_moe_get_block_size(int type) -> int");
|
||||
|
||||
// Mamba selective scan kernel
|
||||
ops.def(
|
||||
"selective_scan_fwd(Tensor! u, Tensor! delta,"
|
||||
@@ -741,12 +713,7 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl("gptq_gemm", TORCH_BOX(&gptq_gemm));
|
||||
ops.impl("gptq_shuffle", TORCH_BOX(&gptq_shuffle));
|
||||
|
||||
// GGML kernels
|
||||
ops.impl("ggml_dequantize", TORCH_BOX(&ggml_dequantize));
|
||||
ops.impl("ggml_mul_mat_vec_a8", TORCH_BOX(&ggml_mul_mat_vec_a8));
|
||||
ops.impl("ggml_mul_mat_a8", TORCH_BOX(&ggml_mul_mat_a8));
|
||||
ops.impl("ggml_moe_a8", TORCH_BOX(&ggml_moe_a8));
|
||||
ops.impl("ggml_moe_a8_vec", TORCH_BOX(&ggml_moe_a8_vec));
|
||||
// Mamba kernels
|
||||
ops.impl("selective_scan_fwd", TORCH_BOX(&selective_scan_fwd));
|
||||
|
||||
ops.impl("paged_attention_v1", TORCH_BOX(&paged_attention_v1));
|
||||
@@ -790,9 +757,6 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CompositeExplicitAutograd, ops) {
|
||||
ops.impl("cutlass_scaled_mm_supports_fp4",
|
||||
TORCH_BOX(&cutlass_scaled_mm_supports_fp4));
|
||||
#endif
|
||||
|
||||
// GGML block size lookup (no tensor args)
|
||||
ops.impl("ggml_moe_get_block_size", TORCH_BOX(&ggml_moe_get_block_size));
|
||||
}
|
||||
|
||||
// Cache ops
|
||||
|
||||
@@ -9,7 +9,6 @@ The following are the supported quantization formats for vLLM:
|
||||
|
||||
- [AutoAWQ](auto_awq.md)
|
||||
- [BitsAndBytes](bnb.md)
|
||||
- [GGUF](gguf.md)
|
||||
- [GPTQModel](gptqmodel.md)
|
||||
- [Intel Neural Compressor](inc.md)
|
||||
- [LLM Compressor](llm_compressor/README.md)
|
||||
|
||||
@@ -3,8 +3,14 @@
|
||||
!!! warning
|
||||
Please note that GGUF support in vLLM is highly experimental and under-optimized at the moment, it might be incompatible with other features. Currently, you can use GGUF as a way to reduce memory footprint. If you encounter any issues, please report them to the vLLM team.
|
||||
|
||||
!!! warning
|
||||
Currently, vllm only supports loading single-file GGUF models. If you have a multi-files GGUF model, you can use [gguf-split](https://github.com/ggerganov/llama.cpp/pull/6135) tool to merge them to a single-file model.
|
||||
!!! note
|
||||
GGUF support has migrated to OOT [vllm-gguf-plugin](https://github.com/vllm-project/vllm-gguf-plugin). Make sure you have GGUF plugin installed before serving a GGUF model.
|
||||
|
||||
Before serving a GGUF model, make sure to install the [vllm-gguf-plugin](https://github.com/vllm-project/vllm-gguf-plugin):
|
||||
|
||||
```bash
|
||||
uv pip install vllm-gguf-plugin
|
||||
```
|
||||
|
||||
To run a GGUF model with vLLM, you can use the `repo_id:quant_type` format to load directly from HuggingFace. For example, to load a Q4_K_M quantized model from [unsloth/Qwen3-0.6B-GGUF](https://huggingface.co/unsloth/Qwen3-0.6B-GGUF):
|
||||
|
||||
|
||||
@@ -32,7 +32,6 @@ def title(text: str) -> str:
|
||||
"mae": "MAE",
|
||||
"ner": "NER",
|
||||
"tpu": "TPU",
|
||||
"gguf": "GGUF",
|
||||
"lora": "LoRA",
|
||||
"nccl": "NCCL",
|
||||
"rlhf": "RLHF",
|
||||
|
||||
@@ -31,7 +31,6 @@ filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/31
|
||||
partial-json-parser # used for parsing partial JSON outputs
|
||||
pyzmq >= 25.0.0
|
||||
msgspec
|
||||
gguf >= 0.17.0
|
||||
mistral_common[image] >= 1.11.3
|
||||
opencv-python-headless >= 4.13.0 # required for video IO
|
||||
pyyaml
|
||||
|
||||
@@ -279,10 +279,6 @@ genai-perf==0.0.16
|
||||
# via -r requirements/test/rocm.in
|
||||
genson==1.3.0
|
||||
# via datamodel-code-generator
|
||||
gguf==0.18.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
google-api-core==2.30.0
|
||||
# via
|
||||
# google-cloud-core
|
||||
@@ -589,7 +585,6 @@ numpy==2.2.6
|
||||
# evaluate
|
||||
# fastparquet
|
||||
# genai-perf
|
||||
# gguf
|
||||
# imagehash
|
||||
# imageio
|
||||
# librosa
|
||||
@@ -959,7 +954,6 @@ pyyaml==6.0.3
|
||||
# datamodel-code-generator
|
||||
# datasets
|
||||
# genai-perf
|
||||
# gguf
|
||||
# huggingface-hub
|
||||
# lm-format-enforcer
|
||||
# optuna
|
||||
@@ -1004,7 +998,6 @@ requests==2.32.5
|
||||
# datasets
|
||||
# docker
|
||||
# evaluate
|
||||
# gguf
|
||||
# google-api-core
|
||||
# google-cloud-storage
|
||||
# gpt-oss
|
||||
@@ -1231,7 +1224,6 @@ tqdm==4.67.3
|
||||
# -r requirements/test/../common.txt
|
||||
# datasets
|
||||
# evaluate
|
||||
# gguf
|
||||
# huggingface-hub
|
||||
# lm-eval
|
||||
# mteb
|
||||
|
||||
@@ -1239,6 +1239,8 @@ setup(
|
||||
"opentelemetry-exporter-otlp>=1.26.0",
|
||||
"opentelemetry-semantic-conventions-ai>=0.4.1",
|
||||
],
|
||||
# extra quantization plugin
|
||||
"extra-quant": ["vllm-gguf-plugin>=0.0.2"],
|
||||
},
|
||||
cmdclass=cmdclass,
|
||||
package_data=package_data,
|
||||
|
||||
@@ -39,12 +39,6 @@ def models_list(*, all: bool = True, keywords: list[str] | None = None):
|
||||
]
|
||||
)
|
||||
|
||||
# TODO: figure out why this fails.
|
||||
if False and is_quant_method_supported("gguf"): # noqa: SIM223
|
||||
TEST_MODELS.append(
|
||||
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF", {"quantization": "gguf"})
|
||||
)
|
||||
|
||||
if is_quant_method_supported("gptq"):
|
||||
TEST_MODELS.append(
|
||||
("TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", {"quantization": "gptq"})
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import gguf
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.utils import opcheck
|
||||
from vllm import _custom_ops as ops # noqa: F401
|
||||
|
||||
|
||||
@pytest.mark.parametrize("quant_type", [12])
|
||||
def test_ggml_opcheck(quant_type):
|
||||
block_size, type_size = gguf.GGML_QUANT_SIZES[quant_type]
|
||||
shape = [256, 1152]
|
||||
qweight = torch.randint(0, 100, shape, device="cuda", dtype=torch.uint8)
|
||||
m = qweight.shape[0]
|
||||
n = qweight.shape[1] // type_size * block_size
|
||||
opcheck(torch.ops._C.ggml_dequantize, (qweight, quant_type, m, n, torch.float16))
|
||||
|
||||
x = torch.rand((m, 512), device="cuda", dtype=torch.float16)
|
||||
opcheck(torch.ops._C.ggml_mul_mat_a8, (qweight, x, quant_type, qweight.shape[0]))
|
||||
opcheck(
|
||||
torch.ops._C.ggml_mul_mat_vec_a8, (qweight, x, quant_type, qweight.shape[0])
|
||||
)
|
||||
|
||||
shape = [256, 1024, 336]
|
||||
qweight = torch.randint(0, 100, shape, device="cuda", dtype=torch.uint8)
|
||||
x = torch.rand((1, 1024), device="cuda", dtype=torch.float16)
|
||||
sorted_token_ids = torch.arange(776, device="cuda")
|
||||
expert_ids = torch.randint(0, 256, (194,), device="cuda")
|
||||
num_tokens_post_padded = torch.tensor([1], dtype=torch.int64, device="cuda")
|
||||
|
||||
opcheck(
|
||||
torch.ops._C.ggml_moe_a8,
|
||||
(
|
||||
x,
|
||||
qweight,
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_padded,
|
||||
quant_type,
|
||||
qweight.shape[0],
|
||||
1,
|
||||
x.shape[0],
|
||||
),
|
||||
)
|
||||
|
||||
topk_ids = torch.zeros((1, 1), device="cuda", dtype=torch.int32)
|
||||
|
||||
opcheck(
|
||||
torch.ops._C.ggml_moe_a8_vec,
|
||||
(x, qweight, topk_ids, 1, quant_type, qweight.shape[0], x.shape[0]),
|
||||
)
|
||||
@@ -1,207 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from gguf import GGMLQuantizationType, GGUFReader, ReaderTensor, dequantize
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.model_executor.layers.fused_moe import fused_experts
|
||||
from vllm.model_executor.layers.quantization.gguf import _fused_moe_gguf
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
GGUF_SAMPLE = snapshot_download("Isotr0py/test-gguf-sample")
|
||||
GGUF_SAMPLE_MOE = snapshot_download("SzymonOzog/test-gguf-moe-sample")
|
||||
|
||||
|
||||
def get_gguf_sample_tensors(
|
||||
hidden_size: int, quant_type: GGMLQuantizationType
|
||||
) -> list[ReaderTensor]:
|
||||
sample_dir = GGUF_SAMPLE
|
||||
filename = f"Quant_{quant_type.name}_{hidden_size}.gguf"
|
||||
sample_file = Path(sample_dir) / filename
|
||||
return GGUFReader(sample_file).tensors
|
||||
|
||||
|
||||
def get_gguf_MoE_tensors(
|
||||
hidden_size: int, quant_type: GGMLQuantizationType
|
||||
) -> list[ReaderTensor]:
|
||||
sample_dir = GGUF_SAMPLE_MOE
|
||||
filename = f"Quant_{quant_type.name}_{hidden_size}.gguf"
|
||||
sample_file = Path(sample_dir) / filename
|
||||
return GGUFReader(sample_file).tensors
|
||||
|
||||
|
||||
DTYPES = [torch.bfloat16] # [torch.half, torch.bfloat16, torch.float32]
|
||||
# Hidden_size for testing, must match the sample file in HF repo,
|
||||
# we have `hidden_size = 256, 1024` for test in HF repo currently.
|
||||
HIDDEN_SIZES = [256, 1024]
|
||||
NUM_TOKENS = [7, 2050] # Arbitrary values for testing
|
||||
SEEDS = [0]
|
||||
QUANT_TYPES = [
|
||||
# i-matrix
|
||||
GGMLQuantizationType.IQ1_M,
|
||||
GGMLQuantizationType.IQ1_S,
|
||||
GGMLQuantizationType.IQ2_S,
|
||||
GGMLQuantizationType.IQ2_XS,
|
||||
GGMLQuantizationType.IQ3_S,
|
||||
GGMLQuantizationType.IQ3_XXS,
|
||||
GGMLQuantizationType.IQ4_NL,
|
||||
GGMLQuantizationType.IQ4_XS,
|
||||
# k-quants
|
||||
GGMLQuantizationType.Q2_K,
|
||||
GGMLQuantizationType.Q3_K,
|
||||
GGMLQuantizationType.Q4_K,
|
||||
GGMLQuantizationType.Q5_K,
|
||||
GGMLQuantizationType.Q6_K,
|
||||
# standard quantization
|
||||
GGMLQuantizationType.Q4_0,
|
||||
GGMLQuantizationType.Q5_0,
|
||||
GGMLQuantizationType.Q8_0,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("quant_type", QUANT_TYPES)
|
||||
@torch.inference_mode()
|
||||
def test_dequantize(
|
||||
hidden_size: int, dtype: torch.dtype, quant_type: GGMLQuantizationType
|
||||
):
|
||||
tensors = get_gguf_sample_tensors(hidden_size, quant_type)
|
||||
for tensor in tensors:
|
||||
shape_str = tensor.name.split("_")[-1]
|
||||
shape = map(int, shape_str.split("x"))
|
||||
|
||||
ref_output = torch.tensor(
|
||||
dequantize(tensor.data, quant_type), device="cuda"
|
||||
).to(dtype)
|
||||
output = ops.ggml_dequantize(
|
||||
torch.tensor(tensor.data, device="cuda"), quant_type, *list(shape), dtype
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=4e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("quant_type", QUANT_TYPES)
|
||||
@torch.inference_mode()
|
||||
def test_mmvq(hidden_size: int, dtype: torch.dtype, quant_type: GGMLQuantizationType):
|
||||
set_random_seed(0)
|
||||
|
||||
tensors = get_gguf_sample_tensors(hidden_size, quant_type)
|
||||
x = torch.rand((1, hidden_size), dtype=dtype, device="cuda")
|
||||
for tensor in tensors:
|
||||
weight = torch.tensor(dequantize(tensor.data, quant_type), device="cuda").to(
|
||||
dtype
|
||||
)
|
||||
ref_output = x @ weight.T
|
||||
|
||||
qweight = torch.tensor(tensor.data, device="cuda")
|
||||
output = ops.ggml_mul_mat_vec_a8(qweight, x, quant_type, qweight.shape[0]).to(
|
||||
dtype
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, ref_output, atol=1, rtol=1e-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize(
|
||||
"quant_type",
|
||||
[
|
||||
# k-quants
|
||||
GGMLQuantizationType.Q2_K,
|
||||
GGMLQuantizationType.Q3_K,
|
||||
GGMLQuantizationType.Q4_K,
|
||||
GGMLQuantizationType.Q5_K,
|
||||
GGMLQuantizationType.Q6_K,
|
||||
# standard quants
|
||||
GGMLQuantizationType.Q4_0,
|
||||
GGMLQuantizationType.Q5_0,
|
||||
GGMLQuantizationType.Q8_0,
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_mmq(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
quant_type: GGMLQuantizationType,
|
||||
):
|
||||
set_random_seed(0)
|
||||
|
||||
tensors = get_gguf_sample_tensors(hidden_size, quant_type)
|
||||
x = torch.rand((num_tokens, hidden_size), dtype=dtype, device="cuda")
|
||||
for tensor in tensors:
|
||||
weight = torch.tensor(dequantize(tensor.data, quant_type), device="cuda").to(
|
||||
dtype
|
||||
)
|
||||
ref_output = x @ weight.T
|
||||
|
||||
qweight = torch.tensor(tensor.data, device="cuda")
|
||||
output = ops.ggml_mul_mat_a8(qweight, x, quant_type, qweight.shape[0])
|
||||
atols = {torch.half: 1, torch.bfloat16: 1.5, torch.float: 1.2}
|
||||
# test matrix has inputs centered around 0 and lower precision from
|
||||
# bfloat16 tends to accumulate and can greatly inflate rtol
|
||||
# since outputs are also very close to 0
|
||||
rtols = {torch.half: 1e-1, torch.bfloat16: 1e4, torch.float: 2e1}
|
||||
torch.testing.assert_close(
|
||||
output, ref_output, atol=atols[dtype], rtol=rtols[dtype]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", [512])
|
||||
@pytest.mark.parametrize("top_k", [4, 8])
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("quant_type", QUANT_TYPES)
|
||||
@torch.inference_mode()
|
||||
def test_moe(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
quant_type: GGMLQuantizationType,
|
||||
top_k: int,
|
||||
):
|
||||
set_random_seed(0)
|
||||
H, E = 1024, 256
|
||||
|
||||
x = torch.rand((num_tokens, H), dtype=dtype, device="cuda")
|
||||
|
||||
topk_weights = torch.rand(num_tokens, top_k, device="cuda", dtype=dtype)
|
||||
topk_ids = torch.randint(
|
||||
0, E, (num_tokens, top_k), device="cuda", dtype=torch.int32
|
||||
)
|
||||
|
||||
tensors = get_gguf_MoE_tensors(hidden_size, quant_type)
|
||||
|
||||
w13 = tensors[0]
|
||||
w2 = tensors[1]
|
||||
|
||||
w13_dequant = torch.tensor(dequantize(w13.data, quant_type), device="cuda").to(
|
||||
dtype
|
||||
)
|
||||
|
||||
w2_dequant = torch.tensor(dequantize(w2.data, quant_type), device="cuda").to(dtype)
|
||||
|
||||
output = _fused_moe_gguf(
|
||||
x,
|
||||
torch.tensor(w13.data, device="cuda"),
|
||||
torch.tensor(w2.data, device="cuda"),
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
quant_type,
|
||||
quant_type,
|
||||
"silu",
|
||||
)
|
||||
|
||||
ref_output = fused_experts(
|
||||
x, w13_dequant, w2_dequant, topk_weights, topk_ids
|
||||
).reshape(output.shape)
|
||||
torch.testing.assert_close(output, ref_output, atol=1, rtol=1e-1)
|
||||
@@ -1,224 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.model_executor.model_loader.gguf_loader import GGUFModelLoader
|
||||
from vllm.model_executor.model_loader.weight_utils import download_gguf
|
||||
|
||||
|
||||
class TestGGUFDownload:
|
||||
"""Test GGUF model downloading functionality."""
|
||||
|
||||
@patch("vllm.model_executor.model_loader.weight_utils.download_weights_from_hf")
|
||||
def test_download_gguf_single_file(self, mock_download):
|
||||
"""Test downloading a single GGUF file."""
|
||||
# Setup mock
|
||||
mock_folder = "/tmp/mock_cache"
|
||||
mock_download.return_value = mock_folder
|
||||
|
||||
# Mock glob to return a single file
|
||||
with patch("glob.glob") as mock_glob:
|
||||
mock_glob.side_effect = lambda pattern, **kwargs: (
|
||||
[f"{mock_folder}/model-IQ1_S.gguf"] if "IQ1_S" in pattern else []
|
||||
)
|
||||
|
||||
result = download_gguf("unsloth/Qwen3-0.6B-GGUF", "IQ1_S")
|
||||
|
||||
# Verify download_weights_from_hf was called with correct patterns
|
||||
mock_download.assert_called_once_with(
|
||||
model_name_or_path="unsloth/Qwen3-0.6B-GGUF",
|
||||
cache_dir=None,
|
||||
allow_patterns=[
|
||||
"*-IQ1_S.gguf",
|
||||
"*-IQ1_S-*.gguf",
|
||||
"*/*-IQ1_S.gguf",
|
||||
"*/*-IQ1_S-*.gguf",
|
||||
],
|
||||
revision=None,
|
||||
ignore_patterns=None,
|
||||
)
|
||||
|
||||
# Verify result is the file path, not folder
|
||||
assert result == f"{mock_folder}/model-IQ1_S.gguf"
|
||||
|
||||
@patch("vllm.model_executor.model_loader.weight_utils.download_weights_from_hf")
|
||||
def test_download_gguf_sharded_files(self, mock_download):
|
||||
"""Test downloading sharded GGUF files."""
|
||||
mock_folder = "/tmp/mock_cache"
|
||||
mock_download.return_value = mock_folder
|
||||
|
||||
# Mock glob to return sharded files
|
||||
with patch("glob.glob") as mock_glob:
|
||||
mock_glob.side_effect = lambda pattern, **kwargs: (
|
||||
[
|
||||
f"{mock_folder}/model-Q2_K-00001-of-00002.gguf",
|
||||
f"{mock_folder}/model-Q2_K-00002-of-00002.gguf",
|
||||
]
|
||||
if "Q2_K" in pattern
|
||||
else []
|
||||
)
|
||||
|
||||
result = download_gguf("unsloth/gpt-oss-120b-GGUF", "Q2_K")
|
||||
|
||||
# Should return the first file after sorting
|
||||
assert result == f"{mock_folder}/model-Q2_K-00001-of-00002.gguf"
|
||||
|
||||
@patch("vllm.model_executor.model_loader.weight_utils.download_weights_from_hf")
|
||||
def test_download_gguf_subdir(self, mock_download):
|
||||
"""Test downloading GGUF files from subdirectory."""
|
||||
mock_folder = "/tmp/mock_cache"
|
||||
mock_download.return_value = mock_folder
|
||||
|
||||
with patch("glob.glob") as mock_glob:
|
||||
mock_glob.side_effect = lambda pattern, **kwargs: (
|
||||
[f"{mock_folder}/Q2_K/model-Q2_K.gguf"]
|
||||
if "Q2_K" in pattern or "**/*.gguf" in pattern
|
||||
else []
|
||||
)
|
||||
|
||||
result = download_gguf("unsloth/gpt-oss-120b-GGUF", "Q2_K")
|
||||
|
||||
assert result == f"{mock_folder}/Q2_K/model-Q2_K.gguf"
|
||||
|
||||
@patch("vllm.model_executor.model_loader.weight_utils.download_weights_from_hf")
|
||||
@patch("glob.glob", return_value=[])
|
||||
def test_download_gguf_no_files_found(self, mock_glob, mock_download):
|
||||
"""Test error when no GGUF files are found."""
|
||||
mock_folder = "/tmp/mock_cache"
|
||||
mock_download.return_value = mock_folder
|
||||
|
||||
with pytest.raises(ValueError, match="Downloaded GGUF files not found"):
|
||||
download_gguf("unsloth/Qwen3-0.6B-GGUF", "IQ1_S")
|
||||
|
||||
|
||||
class TestGGUFModelLoader:
|
||||
"""Test GGUFModelLoader class methods."""
|
||||
|
||||
@patch("os.path.isfile", return_value=True)
|
||||
def test_prepare_weights_local_file(self, mock_isfile):
|
||||
"""Test _prepare_weights with local file."""
|
||||
load_config = LoadConfig(load_format="gguf")
|
||||
loader = GGUFModelLoader(load_config)
|
||||
|
||||
# Create a simple mock ModelConfig with only the model attribute
|
||||
model_config = MagicMock()
|
||||
model_config.model = "/path/to/model.gguf"
|
||||
|
||||
result = loader._prepare_weights(model_config)
|
||||
assert result == "/path/to/model.gguf"
|
||||
mock_isfile.assert_called_once_with("/path/to/model.gguf")
|
||||
|
||||
@patch("vllm.model_executor.model_loader.gguf_loader.hf_hub_download")
|
||||
@patch("os.path.isfile", return_value=False)
|
||||
def test_prepare_weights_repo_filename(self, mock_isfile, mock_hf_download):
|
||||
"""Test _prepare_weights with repo_id/filename.gguf format."""
|
||||
load_config = LoadConfig(load_format="gguf")
|
||||
loader = GGUFModelLoader(load_config)
|
||||
|
||||
mock_hf_download.return_value = "/downloaded/model.gguf"
|
||||
|
||||
model_config = MagicMock()
|
||||
model_config.model = "unsloth/Qwen3-0.6B-GGUF/model.gguf"
|
||||
model_config.revision = "abc123"
|
||||
|
||||
result = loader._prepare_weights(model_config)
|
||||
assert result == "/downloaded/model.gguf"
|
||||
mock_hf_download.assert_called_once_with(
|
||||
repo_id="unsloth/Qwen3-0.6B-GGUF",
|
||||
filename="model.gguf",
|
||||
revision="abc123",
|
||||
cache_dir=None,
|
||||
)
|
||||
|
||||
@patch("vllm.config.model.get_hf_image_processor_config", return_value=None)
|
||||
@patch("vllm.transformers_utils.config.file_or_path_exists", return_value=True)
|
||||
@patch("vllm.config.model.get_config")
|
||||
@patch("vllm.config.model.is_gguf", return_value=True)
|
||||
@patch("vllm.model_executor.model_loader.gguf_loader.download_gguf")
|
||||
@patch("os.path.isfile", return_value=False)
|
||||
def test_prepare_weights_repo_quant_type(
|
||||
self,
|
||||
mock_isfile,
|
||||
mock_download_gguf,
|
||||
mock_is_gguf,
|
||||
mock_get_config,
|
||||
mock_file_exists,
|
||||
mock_get_image_config,
|
||||
):
|
||||
"""Test _prepare_weights with repo_id:quant_type format."""
|
||||
mock_hf_config = MagicMock()
|
||||
mock_hf_config.architectures = ["Qwen3ForCausalLM"]
|
||||
|
||||
class MockTextConfig:
|
||||
max_position_embeddings = 4096
|
||||
sliding_window = None
|
||||
model_type = "qwen3"
|
||||
num_attention_heads = 32
|
||||
|
||||
mock_text_config = MockTextConfig()
|
||||
mock_hf_config.get_text_config.return_value = mock_text_config
|
||||
mock_hf_config.dtype = "bfloat16"
|
||||
mock_get_config.return_value = mock_hf_config
|
||||
|
||||
load_config = LoadConfig(load_format="gguf")
|
||||
loader = GGUFModelLoader(load_config)
|
||||
|
||||
mock_download_gguf.return_value = "/downloaded/model-IQ1_S.gguf"
|
||||
|
||||
model_config = ModelConfig(
|
||||
model="unsloth/Qwen3-0.6B-GGUF:IQ1_S", tokenizer="Qwen/Qwen3-0.6B"
|
||||
)
|
||||
result = loader._prepare_weights(model_config)
|
||||
# The actual result will be the downloaded file path from mock
|
||||
assert result == "/downloaded/model-IQ1_S.gguf"
|
||||
mock_download_gguf.assert_called_once_with(
|
||||
"unsloth/Qwen3-0.6B-GGUF",
|
||||
"IQ1_S",
|
||||
cache_dir=None,
|
||||
revision=None,
|
||||
ignore_patterns=["original/**/*"],
|
||||
)
|
||||
|
||||
@patch("vllm.config.model.get_hf_image_processor_config", return_value=None)
|
||||
@patch("vllm.config.model.get_config")
|
||||
@patch("vllm.config.model.is_gguf", return_value=False)
|
||||
@patch("vllm.transformers_utils.gguf_utils.check_gguf_file", return_value=False)
|
||||
@patch("os.path.isfile", return_value=False)
|
||||
def test_prepare_weights_invalid_format(
|
||||
self,
|
||||
mock_isfile,
|
||||
mock_check_gguf,
|
||||
mock_is_gguf,
|
||||
mock_get_config,
|
||||
mock_get_image_config,
|
||||
):
|
||||
"""Test _prepare_weights with invalid format."""
|
||||
mock_hf_config = MagicMock()
|
||||
mock_hf_config.architectures = ["Qwen3ForCausalLM"]
|
||||
|
||||
class MockTextConfig:
|
||||
max_position_embeddings = 4096
|
||||
sliding_window = None
|
||||
model_type = "qwen3"
|
||||
num_attention_heads = 32
|
||||
|
||||
mock_text_config = MockTextConfig()
|
||||
mock_hf_config.get_text_config.return_value = mock_text_config
|
||||
mock_hf_config.dtype = "bfloat16"
|
||||
mock_get_config.return_value = mock_hf_config
|
||||
|
||||
load_config = LoadConfig(load_format="gguf")
|
||||
loader = GGUFModelLoader(load_config)
|
||||
|
||||
# Create ModelConfig with a valid repo_id to avoid validation errors
|
||||
# Then test _prepare_weights with invalid format
|
||||
model_config = ModelConfig(model="unsloth/Qwen3-0.6B")
|
||||
# Manually set model to invalid format after creation
|
||||
model_config.model = "invalid-format"
|
||||
with pytest.raises(ValueError, match="Unrecognised GGUF reference"):
|
||||
loader._prepare_weights(model_config)
|
||||
+13
-85
@@ -1,23 +1,18 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests gguf models against unquantized models generations
|
||||
Note: To pass the test, quantization higher than Q4 should be used
|
||||
E2E tests for GGUF plugin functionality.
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import NamedTuple
|
||||
|
||||
import pytest
|
||||
from huggingface_hub import hf_hub_download
|
||||
from pytest import MarkDecorator
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
|
||||
from ...conftest import VllmRunner
|
||||
from ...models.utils import check_logprobs_close
|
||||
from ...utils import multi_gpu_test
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "true"
|
||||
|
||||
@@ -26,80 +21,24 @@ MAX_MODEL_LEN = 1024
|
||||
|
||||
class GGUFTestConfig(NamedTuple):
|
||||
original_model: str
|
||||
gguf_repo: str
|
||||
gguf_filename: str
|
||||
marks: list[MarkDecorator] = []
|
||||
gguf_model_path: str # Full path to .gguf file
|
||||
|
||||
@property
|
||||
def gguf_model(self):
|
||||
return hf_hub_download(self.gguf_repo, filename=self.gguf_filename)
|
||||
|
||||
|
||||
LLAMA_CONFIG = GGUFTestConfig(
|
||||
original_model="meta-llama/Llama-3.2-1B-Instruct",
|
||||
gguf_repo="bartowski/Llama-3.2-1B-Instruct-GGUF",
|
||||
gguf_filename="Llama-3.2-1B-Instruct-Q6_K.gguf",
|
||||
)
|
||||
|
||||
QWEN2_CONFIG = GGUFTestConfig(
|
||||
original_model="Qwen/Qwen2.5-1.5B-Instruct",
|
||||
gguf_repo="Qwen/Qwen2.5-1.5B-Instruct-GGUF",
|
||||
gguf_filename="qwen2.5-1.5b-instruct-q6_k.gguf",
|
||||
)
|
||||
|
||||
QWEN3_CONFIG = GGUFTestConfig(
|
||||
original_model="Qwen/Qwen3-0.6B",
|
||||
gguf_repo="unsloth/Qwen3-0.6B-GGUF",
|
||||
gguf_filename="Qwen3-0.6B-BF16.gguf",
|
||||
gguf_model_path="unsloth/Qwen3-0.6B-GGUF:Q8_0",
|
||||
)
|
||||
|
||||
PHI3_CONFIG = GGUFTestConfig(
|
||||
original_model="microsoft/Phi-3.5-mini-instruct",
|
||||
gguf_repo="bartowski/Phi-3.5-mini-instruct-GGUF",
|
||||
gguf_filename="Phi-3.5-mini-instruct-IQ4_XS.gguf",
|
||||
|
||||
OLMOE_CONFIG = GGUFTestConfig(
|
||||
original_model="allenai/OLMoE-1B-7B-0125",
|
||||
gguf_model_path="allenai/OLMoE-1B-7B-0125-GGUF:Q6_K",
|
||||
)
|
||||
|
||||
GPT2_CONFIG = GGUFTestConfig(
|
||||
original_model="openai-community/gpt2-large",
|
||||
gguf_repo="QuantFactory/gpt2-large-GGUF",
|
||||
gguf_filename="gpt2-large.Q4_K_M.gguf",
|
||||
)
|
||||
|
||||
STABLELM_CONFIG = GGUFTestConfig(
|
||||
original_model="stabilityai/stablelm-3b-4e1t",
|
||||
gguf_repo="afrideva/stablelm-3b-4e1t-GGUF",
|
||||
gguf_filename="stablelm-3b-4e1t.q4_k_m.gguf",
|
||||
)
|
||||
|
||||
STARCODER_CONFIG = GGUFTestConfig(
|
||||
original_model="bigcode/starcoder2-3b",
|
||||
gguf_repo="QuantFactory/starcoder2-3b-GGUF",
|
||||
gguf_filename="starcoder2-3b.Q6_K.gguf",
|
||||
)
|
||||
|
||||
DOLPHIN_CONFIG = GGUFTestConfig(
|
||||
# Test VocabParallelEmbedding sharding issue.
|
||||
original_model="cognitivecomputations/TinyDolphin-2.8-1.1b",
|
||||
gguf_repo="tsunemoto/TinyDolphin-2.8-1.1b-GGUF",
|
||||
gguf_filename="tinydolphin-2.8-1.1b.Q6_K.gguf",
|
||||
)
|
||||
|
||||
GEMMA3_CONFIG = GGUFTestConfig(
|
||||
original_model="google/gemma-3-270m-it",
|
||||
gguf_repo="ggml-org/gemma-3-270m-it-qat-GGUF",
|
||||
gguf_filename="gemma-3-270m-it-qat-Q4_0.gguf",
|
||||
)
|
||||
|
||||
MODELS = [
|
||||
# LLAMA_CONFIG, # broken: https://github.com/vllm-project/vllm/issues/19458
|
||||
QWEN2_CONFIG,
|
||||
QWEN3_CONFIG,
|
||||
PHI3_CONFIG,
|
||||
GPT2_CONFIG,
|
||||
STABLELM_CONFIG,
|
||||
DOLPHIN_CONFIG,
|
||||
GEMMA3_CONFIG,
|
||||
# STARCODER_CONFIG, # broken
|
||||
OLMOE_CONFIG,
|
||||
]
|
||||
|
||||
|
||||
@@ -121,7 +60,7 @@ def check_model_outputs(
|
||||
|
||||
# Run gguf model.
|
||||
with vllm_runner(
|
||||
model_name=model.gguf_model,
|
||||
model_name=model.gguf_model_path,
|
||||
enforce_eager=True,
|
||||
tokenizer_name=model.original_model,
|
||||
dtype=dtype,
|
||||
@@ -154,17 +93,10 @@ def check_model_outputs(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("gguf"),
|
||||
reason="gguf is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[pytest.param(test_config, marks=test_config.marks) for test_config in MODELS],
|
||||
)
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@pytest.mark.parametrize("max_tokens", [32])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@pytest.mark.parametrize("num_logprobs", [8])
|
||||
@pytest.mark.parametrize("tp_size", [1])
|
||||
def test_models(
|
||||
vllm_runner: type[VllmRunner],
|
||||
@@ -180,11 +112,7 @@ def test_models(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("gguf"),
|
||||
reason="gguf is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("model", [LLAMA_CONFIG])
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", [8])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
+6
-19
@@ -12,13 +12,12 @@ from huggingface_hub import hf_hub_download
|
||||
from pytest import MarkDecorator
|
||||
from transformers import AutoModelForImageTextToText
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.multimodal.image import rescale_image_size
|
||||
from vllm.utils.torch_utils import set_default_torch_num_threads
|
||||
|
||||
from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner
|
||||
from ...utils import check_logprobs_close
|
||||
from ...conftest import IMAGE_ASSETS, HfRunner, VllmRunner
|
||||
from ...models.utils import check_logprobs_close
|
||||
|
||||
|
||||
class GGUFMMTestConfig(NamedTuple):
|
||||
@@ -66,20 +65,18 @@ GEMMA3_CONFIG = GGUFMMTestConfig(
|
||||
prompt=_GEMMA3_PROMPTS,
|
||||
image_names=_GEMMA3_IMAGE_NAMES,
|
||||
max_model_len=4096,
|
||||
marks=[pytest.mark.core_model],
|
||||
mm_processor_kwargs={},
|
||||
)
|
||||
|
||||
# Pan-and-scan multimodal - uses unquantized BF16 GGUF
|
||||
GEMMA3_CONFIG_PAN_AND_SCAN = GGUFMMTestConfig(
|
||||
original_model="google/gemma-3-4b-it",
|
||||
gguf_repo="unsloth/gemma-3-4b-it-GGUF",
|
||||
gguf_backbone="gemma-3-4b-it-BF16.gguf",
|
||||
gguf_mmproj="mmproj-BF16.gguf",
|
||||
gguf_repo="google/gemma-3-4b-it-qat-q4_0-gguf",
|
||||
gguf_backbone="gemma-3-4b-it-q4_0.gguf",
|
||||
gguf_mmproj="mmproj-model-f16-4B.gguf",
|
||||
prompt=_GEMMA3_PROMPTS,
|
||||
image_names=_GEMMA3_IMAGE_NAMES,
|
||||
max_model_len=4096,
|
||||
marks=[pytest.mark.core_model],
|
||||
mm_processor_kwargs={"do_pan_and_scan": True},
|
||||
)
|
||||
|
||||
@@ -153,17 +150,7 @@ def run_multimodal_gguf_test(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("gguf"),
|
||||
reason="gguf is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
pytest.param(test_config, marks=test_config.marks)
|
||||
for test_config in MODELS_TO_TEST
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model", MODELS_TO_TEST)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@pytest.mark.parametrize("max_tokens", [32])
|
||||
@pytest.mark.parametrize("num_logprobs", [10])
|
||||
@@ -1,15 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.transformers_utils.gguf_utils import (
|
||||
is_gguf,
|
||||
is_remote_gguf,
|
||||
split_remote_gguf,
|
||||
)
|
||||
from vllm.transformers_utils.utils import (
|
||||
is_azure,
|
||||
is_cloud_storage,
|
||||
@@ -45,203 +35,3 @@ def test_is_cloud_storage():
|
||||
assert is_cloud_storage("az://model-container/path")
|
||||
assert not is_cloud_storage("/unix/local/path")
|
||||
assert not is_cloud_storage("nfs://nfs-fqdn.local")
|
||||
|
||||
|
||||
class TestIsRemoteGGUF:
|
||||
"""Test is_remote_gguf utility function."""
|
||||
|
||||
def test_is_remote_gguf_with_colon_and_slash(self):
|
||||
"""Test is_remote_gguf with repo_id:quant_type format."""
|
||||
# Valid quant types (exact GGML types)
|
||||
assert is_remote_gguf("unsloth/Qwen3-0.6B-GGUF:IQ1_S")
|
||||
assert is_remote_gguf("user/repo:Q2_K")
|
||||
assert is_remote_gguf("repo/model:Q4_K")
|
||||
assert is_remote_gguf("repo/model:Q8_0")
|
||||
|
||||
# Invalid quant types should return False
|
||||
assert not is_remote_gguf("repo/model:quant")
|
||||
assert not is_remote_gguf("repo/model:INVALID")
|
||||
assert not is_remote_gguf("repo/model:invalid_type")
|
||||
|
||||
def test_is_remote_gguf_extended_quant_types(self):
|
||||
"""Test is_remote_gguf with extended quant type naming conventions."""
|
||||
# Extended quant types with _M, _S, _L suffixes
|
||||
assert is_remote_gguf("repo/model:Q4_K_M")
|
||||
assert is_remote_gguf("repo/model:Q4_K_S")
|
||||
assert is_remote_gguf("repo/model:Q3_K_L")
|
||||
assert is_remote_gguf("repo/model:Q5_K_M")
|
||||
assert is_remote_gguf("repo/model:Q3_K_S")
|
||||
|
||||
# Extended quant types with _XL, _XS, _XXS suffixes
|
||||
assert is_remote_gguf("repo/model:Q5_K_XL")
|
||||
assert is_remote_gguf("repo/model:IQ4_XS")
|
||||
assert is_remote_gguf("repo/model:IQ3_XXS")
|
||||
|
||||
# Invalid extended types (base type doesn't exist)
|
||||
assert not is_remote_gguf("repo/model:INVALID_M")
|
||||
assert not is_remote_gguf("repo/model:Q9_K_M")
|
||||
|
||||
def test_is_remote_gguf_nonstandard_quant_type(self):
|
||||
"""Test is_remote_gguf with non-standard quant types containing
|
||||
a known GGML type."""
|
||||
# Non-standard quant types with known GGML type after prefix
|
||||
assert is_remote_gguf("unsloth/Qwen3.5-35B-A3B-GGUF:UD-Q4_K_XL")
|
||||
assert is_remote_gguf("user/Model:UD-Q4_K_M")
|
||||
assert is_remote_gguf("user/SomeModel:Custom-Q8_0")
|
||||
|
||||
# Exact GGML type after prefix (no suffix stripping needed)
|
||||
assert is_remote_gguf("user/Model-GGUF:UD-IQ4_NL")
|
||||
assert is_remote_gguf("user/Model-GGUF:UD-Q8_0")
|
||||
|
||||
# Completely unknown quant types should still fail
|
||||
assert not is_remote_gguf("repo/model:TOTALLY-RANDOM")
|
||||
assert not is_remote_gguf("user/Model:UD-INVALID")
|
||||
|
||||
# No dash separator → not recognized as prefixed
|
||||
assert not is_remote_gguf("repo/model:UDIQ4NL")
|
||||
|
||||
def test_is_remote_gguf_without_colon(self):
|
||||
"""Test is_remote_gguf without colon."""
|
||||
assert not is_remote_gguf("repo/model")
|
||||
assert not is_remote_gguf("unsloth/Qwen3-0.6B-GGUF")
|
||||
|
||||
def test_is_remote_gguf_without_slash(self):
|
||||
"""Test is_remote_gguf without slash."""
|
||||
assert not is_remote_gguf("model.gguf")
|
||||
# Even with valid quant_type, no slash means not remote GGUF
|
||||
assert not is_remote_gguf("model:IQ1_S")
|
||||
assert not is_remote_gguf("model:quant")
|
||||
|
||||
def test_is_remote_gguf_local_path(self):
|
||||
"""Test is_remote_gguf with local file path."""
|
||||
assert not is_remote_gguf("/path/to/model.gguf")
|
||||
assert not is_remote_gguf("./model.gguf")
|
||||
|
||||
def test_is_remote_gguf_with_path_object(self):
|
||||
"""Test is_remote_gguf with Path object."""
|
||||
assert is_remote_gguf(Path("unsloth/Qwen3-0.6B-GGUF:IQ1_S"))
|
||||
assert not is_remote_gguf(Path("repo/model"))
|
||||
|
||||
def test_is_remote_gguf_with_http_https(self):
|
||||
"""Test is_remote_gguf with HTTP/HTTPS URLs."""
|
||||
# HTTP/HTTPS URLs should return False even with valid quant_type
|
||||
assert not is_remote_gguf("http://example.com/repo/model:IQ1_S")
|
||||
assert not is_remote_gguf("https://huggingface.co/repo/model:Q2_K")
|
||||
assert not is_remote_gguf("http://repo/model:Q4_K")
|
||||
assert not is_remote_gguf("https://repo/model:Q8_0")
|
||||
|
||||
def test_is_remote_gguf_with_cloud_storage(self):
|
||||
"""Test is_remote_gguf with cloud storage paths."""
|
||||
# Cloud storage paths should return False even with valid quant_type
|
||||
assert not is_remote_gguf("s3://bucket/repo/model:IQ1_S")
|
||||
assert not is_remote_gguf("gs://bucket/repo/model:Q2_K")
|
||||
assert not is_remote_gguf("s3://repo/model:Q4_K")
|
||||
assert not is_remote_gguf("gs://repo/model:Q8_0")
|
||||
|
||||
|
||||
class TestSplitRemoteGGUF:
|
||||
"""Test split_remote_gguf utility function."""
|
||||
|
||||
def test_split_remote_gguf_valid(self):
|
||||
"""Test split_remote_gguf with valid repo_id:quant_type format."""
|
||||
repo_id, quant_type = split_remote_gguf("unsloth/Qwen3-0.6B-GGUF:IQ1_S")
|
||||
assert repo_id == "unsloth/Qwen3-0.6B-GGUF"
|
||||
assert quant_type == "IQ1_S"
|
||||
|
||||
repo_id, quant_type = split_remote_gguf("repo/model:Q2_K")
|
||||
assert repo_id == "repo/model"
|
||||
assert quant_type == "Q2_K"
|
||||
|
||||
def test_split_remote_gguf_extended_quant_types(self):
|
||||
"""Test split_remote_gguf with extended quant type naming conventions."""
|
||||
repo_id, quant_type = split_remote_gguf("unsloth/Qwen3-0.6B-GGUF:Q4_K_M")
|
||||
assert repo_id == "unsloth/Qwen3-0.6B-GGUF"
|
||||
assert quant_type == "Q4_K_M"
|
||||
|
||||
repo_id, quant_type = split_remote_gguf("repo/model:Q3_K_S")
|
||||
assert repo_id == "repo/model"
|
||||
assert quant_type == "Q3_K_S"
|
||||
|
||||
def test_split_remote_gguf_nonstandard_quant_type(self):
|
||||
"""Test split_remote_gguf with non-standard quant types in GGUF repos."""
|
||||
repo_id, quant_type = split_remote_gguf(
|
||||
"unsloth/Qwen3.5-35B-A3B-GGUF:UD-Q4_K_XL"
|
||||
)
|
||||
assert repo_id == "unsloth/Qwen3.5-35B-A3B-GGUF"
|
||||
assert quant_type == "UD-Q4_K_XL"
|
||||
|
||||
def test_split_remote_gguf_with_path_object(self):
|
||||
"""Test split_remote_gguf with Path object."""
|
||||
repo_id, quant_type = split_remote_gguf(Path("unsloth/Qwen3-0.6B-GGUF:IQ1_S"))
|
||||
assert repo_id == "unsloth/Qwen3-0.6B-GGUF"
|
||||
assert quant_type == "IQ1_S"
|
||||
|
||||
def test_split_remote_gguf_invalid(self):
|
||||
"""Test split_remote_gguf with invalid format."""
|
||||
# Invalid format (no colon) - is_remote_gguf returns False
|
||||
with pytest.raises(ValueError, match="Wrong GGUF model"):
|
||||
split_remote_gguf("repo/model")
|
||||
|
||||
# Invalid quant type - is_remote_gguf returns False
|
||||
with pytest.raises(ValueError, match="Wrong GGUF model"):
|
||||
split_remote_gguf("repo/model:INVALID_TYPE")
|
||||
|
||||
# HTTP URL - is_remote_gguf returns False
|
||||
with pytest.raises(ValueError, match="Wrong GGUF model"):
|
||||
split_remote_gguf("http://repo/model:IQ1_S")
|
||||
|
||||
# Cloud storage - is_remote_gguf returns False
|
||||
with pytest.raises(ValueError, match="Wrong GGUF model"):
|
||||
split_remote_gguf("s3://bucket/repo/model:Q2_K")
|
||||
|
||||
|
||||
class TestIsGGUF:
|
||||
"""Test is_gguf utility function."""
|
||||
|
||||
@patch("vllm.transformers_utils.gguf_utils.check_gguf_file", return_value=True)
|
||||
def test_is_gguf_with_local_file(self, mock_check_gguf):
|
||||
"""Test is_gguf with local GGUF file."""
|
||||
assert is_gguf("/path/to/model.gguf")
|
||||
assert is_gguf("./model.gguf")
|
||||
|
||||
def test_is_gguf_with_remote_gguf(self):
|
||||
"""Test is_gguf with remote GGUF format."""
|
||||
# Valid remote GGUF format (repo_id:quant_type with valid quant_type)
|
||||
assert is_gguf("unsloth/Qwen3-0.6B-GGUF:IQ1_S")
|
||||
assert is_gguf("repo/model:Q2_K")
|
||||
assert is_gguf("repo/model:Q4_K")
|
||||
|
||||
# Extended quant types with suffixes
|
||||
assert is_gguf("repo/model:Q4_K_M")
|
||||
assert is_gguf("repo/model:Q3_K_S")
|
||||
assert is_gguf("repo/model:Q5_K_L")
|
||||
|
||||
# Invalid quant_type should return False
|
||||
assert not is_gguf("repo/model:quant")
|
||||
assert not is_gguf("repo/model:INVALID")
|
||||
|
||||
@patch("vllm.transformers_utils.gguf_utils.check_gguf_file", return_value=False)
|
||||
def test_is_gguf_false(self, mock_check_gguf):
|
||||
"""Test is_gguf returns False for non-GGUF models."""
|
||||
assert not is_gguf("unsloth/Qwen3-0.6B")
|
||||
assert not is_gguf("repo/model")
|
||||
assert not is_gguf("model")
|
||||
|
||||
def test_is_gguf_edge_cases(self):
|
||||
"""Test is_gguf with edge cases."""
|
||||
# Empty string
|
||||
assert not is_gguf("")
|
||||
|
||||
# Only colon, no slash (even with valid quant_type)
|
||||
assert not is_gguf("model:IQ1_S")
|
||||
|
||||
# Only slash, no colon
|
||||
assert not is_gguf("repo/model")
|
||||
|
||||
# HTTP/HTTPS URLs
|
||||
assert not is_gguf("http://repo/model:IQ1_S")
|
||||
assert not is_gguf("https://repo/model:Q2_K")
|
||||
|
||||
# Cloud storage
|
||||
assert not is_gguf("s3://bucket/repo/model:IQ1_S")
|
||||
assert not is_gguf("gs://bucket/repo/model:Q2_K")
|
||||
|
||||
@@ -768,69 +768,6 @@ if hasattr(torch.ops._C, "allspark_w8a16_gemm"):
|
||||
return torch.empty((m, n), device=a.device, dtype=a.dtype)
|
||||
|
||||
|
||||
if hasattr(torch.ops._C, "ggml_dequantize"):
|
||||
|
||||
@register_fake("_C::ggml_dequantize")
|
||||
def _ggml_dequantize_fake(
|
||||
W: torch.Tensor,
|
||||
quant_type: int,
|
||||
m: torch.SymInt,
|
||||
n: torch.SymInt,
|
||||
dtype: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty((m, n), dtype=torch.float16, device=W.device)
|
||||
|
||||
@register_fake("_C::ggml_mul_mat_vec_a8")
|
||||
def _ggml_mul_mat_vec_a8_fake(
|
||||
W: torch.Tensor,
|
||||
X: torch.Tensor,
|
||||
quant_type: int,
|
||||
row: torch.SymInt,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty((X.shape[0], row), dtype=X.dtype, device=W.device)
|
||||
|
||||
@register_fake("_C::ggml_mul_mat_a8")
|
||||
def _ggml_mul_mat_a8_fake(
|
||||
W: torch.Tensor,
|
||||
X: torch.Tensor,
|
||||
quant_type: int,
|
||||
row: torch.SymInt,
|
||||
) -> torch.Tensor:
|
||||
batch = X.size(0)
|
||||
return torch.empty((batch, row), dtype=X.dtype, device=W.device)
|
||||
|
||||
@register_fake("_C::ggml_moe_a8")
|
||||
def _ggml_moe_a8_fake(
|
||||
X: torch.Tensor,
|
||||
W: torch.Tensor,
|
||||
sorted_token_ids: torch.Tensor,
|
||||
expert_ids: torch.Tensor,
|
||||
num_tokens_post_padded: torch.Tensor,
|
||||
quant_type: int,
|
||||
row: torch.SymInt,
|
||||
top_k: torch.SymInt,
|
||||
tokens: torch.SymInt,
|
||||
) -> torch.Tensor:
|
||||
tokens = X.size(0)
|
||||
return torch.empty((tokens * top_k, row), dtype=torch.float16, device=W.device)
|
||||
|
||||
|
||||
if hasattr(torch.ops._C, "ggml_moe_a8_vec"):
|
||||
|
||||
@register_fake("_C::ggml_moe_a8_vec")
|
||||
def _ggml_moe_a8_vec_fake(
|
||||
X: torch.Tensor,
|
||||
W: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
top_k: int,
|
||||
quant_type: int,
|
||||
row: torch.SymInt,
|
||||
tokens: torch.SymInt,
|
||||
) -> torch.Tensor:
|
||||
tokens = X.size(0)
|
||||
return torch.empty((tokens * top_k, row), dtype=X.dtype, device=W.device)
|
||||
|
||||
|
||||
# cutlass
|
||||
def cutlass_scaled_mm_supports_fp4(cuda_device_capability: int) -> bool:
|
||||
return torch.ops._C.cutlass_scaled_mm_supports_fp4(cuda_device_capability)
|
||||
@@ -2195,71 +2132,6 @@ def scaled_int8_quant(
|
||||
return output, input_scales, input_azp
|
||||
|
||||
|
||||
# gguf
|
||||
def ggml_dequantize(
|
||||
W: torch.Tensor, quant_type: int, m: int, n: int, dtype: torch.dtype | None
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._C.ggml_dequantize(W, quant_type, m, n, dtype)
|
||||
|
||||
|
||||
def ggml_mul_mat_vec_a8(
|
||||
W: torch.Tensor,
|
||||
X: torch.Tensor,
|
||||
quant_type: int,
|
||||
row: int,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._C.ggml_mul_mat_vec_a8(W, X, quant_type, row)
|
||||
|
||||
|
||||
def ggml_mul_mat_a8(
|
||||
W: torch.Tensor,
|
||||
X: torch.Tensor,
|
||||
quant_type: int,
|
||||
row: int,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._C.ggml_mul_mat_a8(W, X, quant_type, row)
|
||||
|
||||
|
||||
def ggml_moe_a8(
|
||||
X: torch.Tensor,
|
||||
W: torch.Tensor,
|
||||
sorted_token_ids: torch.Tensor,
|
||||
expert_ids: torch.Tensor,
|
||||
num_tokens_post_padded: torch.Tensor,
|
||||
quant_type: int,
|
||||
row: int,
|
||||
top_k: int,
|
||||
tokens: int,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._C.ggml_moe_a8(
|
||||
X,
|
||||
W,
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_padded,
|
||||
quant_type,
|
||||
row,
|
||||
top_k,
|
||||
tokens,
|
||||
)
|
||||
|
||||
|
||||
def ggml_moe_a8_vec(
|
||||
X: torch.Tensor,
|
||||
W: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
top_k: int,
|
||||
quant_type: int,
|
||||
row: torch.SymInt,
|
||||
tokens: torch.SymInt,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._C.ggml_moe_a8_vec(X, W, topk_ids, top_k, quant_type, row, tokens)
|
||||
|
||||
|
||||
def ggml_moe_get_block_size(quant_type: int) -> int:
|
||||
return torch.ops._C.ggml_moe_get_block_size(quant_type)
|
||||
|
||||
|
||||
# mamba
|
||||
def selective_scan_fwd(
|
||||
u: torch.Tensor,
|
||||
|
||||
@@ -51,8 +51,6 @@ class LoadConfig:
|
||||
- "bitsandbytes" will load the weights using bitsandbytes quantization.
|
||||
- "sharded_state" will load weights from pre-sharded checkpoint files,
|
||||
supporting efficient loading of tensor-parallel models.
|
||||
- "gguf" will load weights from GGUF format files (details specified in
|
||||
https://github.com/ggml-org/ggml/blob/master/docs/gguf.md).
|
||||
- "mistral" will load weights from consolidated safetensors files used by
|
||||
Mistral models.
|
||||
- "modelexpress" will load weights using ModelExpress.
|
||||
|
||||
+1
-24
@@ -42,12 +42,6 @@ from vllm.transformers_utils.config import (
|
||||
uses_mrope,
|
||||
uses_xdrope_dim,
|
||||
)
|
||||
from vllm.transformers_utils.gguf_utils import (
|
||||
is_gguf,
|
||||
is_remote_gguf,
|
||||
maybe_patch_hf_config_from_gguf,
|
||||
split_remote_gguf,
|
||||
)
|
||||
from vllm.transformers_utils.model_arch_config_convertor import (
|
||||
MODEL_ARCH_CONFIG_CONVERTORS,
|
||||
ModelArchConfigConvertorBase,
|
||||
@@ -547,11 +541,6 @@ class ModelConfig:
|
||||
hf_overrides_fn=hf_overrides_fn,
|
||||
token=self.hf_token,
|
||||
)
|
||||
hf_config = maybe_patch_hf_config_from_gguf(
|
||||
self.model,
|
||||
hf_config,
|
||||
)
|
||||
|
||||
self.hf_config = hf_config
|
||||
if dict_overrides:
|
||||
self._apply_dict_overrides(hf_config, dict_overrides)
|
||||
@@ -724,14 +713,6 @@ class ModelConfig:
|
||||
"disable the cache with --mm-processor-cache-gb 0."
|
||||
)
|
||||
|
||||
# Multimodal GGUF models must use original repo for mm processing
|
||||
if is_gguf(self.tokenizer) and self.is_multimodal_model:
|
||||
raise ValueError(
|
||||
"Loading a multimodal GGUF model needs to use original "
|
||||
"tokenizer. Please specify the unquantized hf model's "
|
||||
"repo name or path using the --tokenizer argument."
|
||||
)
|
||||
|
||||
if self.disable_sliding_window:
|
||||
# Set after get_and_verify_max_len to ensure that max_model_len
|
||||
# can be correctly capped to sliding window size
|
||||
@@ -884,10 +865,7 @@ class ModelConfig:
|
||||
self.tokenizer = object_storage_tokenizer.dir
|
||||
|
||||
def _get_encoder_config(self) -> dict[str, Any] | None:
|
||||
model = self.model
|
||||
if is_remote_gguf(model):
|
||||
model, _ = split_remote_gguf(model)
|
||||
return get_sentence_transformer_tokenizer_config(model, self.revision)
|
||||
return get_sentence_transformer_tokenizer_config(self.model, self.revision)
|
||||
|
||||
def _get_default_runner_type(
|
||||
self,
|
||||
@@ -1019,7 +997,6 @@ class ModelConfig:
|
||||
"gpt_oss_mxfp4",
|
||||
"deepseek_v4_fp8",
|
||||
"humming",
|
||||
"gguf",
|
||||
]
|
||||
# if the user specifies humming, we should always use humming
|
||||
if self.quantization == "humming":
|
||||
|
||||
@@ -103,7 +103,6 @@ from vllm.transformers_utils.config import (
|
||||
is_interleaved,
|
||||
maybe_override_with_speculators,
|
||||
)
|
||||
from vllm.transformers_utils.gguf_utils import is_gguf
|
||||
from vllm.transformers_utils.repo_utils import get_model_path
|
||||
from vllm.transformers_utils.utils import is_cloud_storage
|
||||
from vllm.utils.argparse_utils import (
|
||||
@@ -1558,10 +1557,6 @@ class EngineArgs:
|
||||
return engine_args
|
||||
|
||||
def create_model_config(self) -> ModelConfig:
|
||||
# gguf file needs a specific model loader
|
||||
if is_gguf(self.model):
|
||||
self.quantization = self.load_format = "gguf"
|
||||
|
||||
if not envs.VLLM_ENABLE_V1_MULTIPROCESSING:
|
||||
logger.warning(
|
||||
"The global random seed is set to %d. Since "
|
||||
|
||||
@@ -6,7 +6,6 @@ from enum import Enum
|
||||
from typing import TYPE_CHECKING, Any, Literal, cast, overload
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import UninitializedParameter
|
||||
|
||||
from vllm.distributed.eplb.eplb_state import EplbState
|
||||
from vllm.logger import init_logger
|
||||
@@ -625,13 +624,6 @@ class RoutedExperts(PluggableLayer):
|
||||
# dimension intermediate_size_per_partition is used.
|
||||
SHARD_ID_TO_SHARDED_DIM = {"w1": 0, "w2": 1, "w3": 0}
|
||||
|
||||
is_gguf_weight = getattr(param, "is_gguf_weight", False)
|
||||
is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
|
||||
if is_gguf_weight_type:
|
||||
param.weight_type = loaded_weight.item()
|
||||
param.data.copy_(loaded_weight)
|
||||
return True if return_success else None
|
||||
|
||||
# Case for BitsAndBytes
|
||||
use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)
|
||||
if use_bitsandbytes_4bit:
|
||||
@@ -677,18 +669,6 @@ class RoutedExperts(PluggableLayer):
|
||||
if full_load:
|
||||
shard_dim += 1
|
||||
|
||||
# Materialize GGUF UninitializedParameter accounting merged weights
|
||||
if is_gguf_weight and isinstance(param, UninitializedParameter):
|
||||
# To materialize a tensor, we must have full shape including
|
||||
# number of experts, making this portion to require `full_load`.
|
||||
assert full_load
|
||||
final_shape = list(loaded_weight.shape)
|
||||
# w1 and w3 are merged per expert.
|
||||
if shard_id in {"w1", "w3"}:
|
||||
final_shape[1] *= 2
|
||||
final_shape[shard_dim] = final_shape[shard_dim] // self.moe_config.tp_size
|
||||
param.materialize(final_shape, dtype=loaded_weight.dtype)
|
||||
|
||||
expert_data = param.data if full_load else param.data[expert_id]
|
||||
|
||||
# Case input scale: input_scale loading is only supported for fp8
|
||||
|
||||
@@ -5,7 +5,7 @@ import itertools
|
||||
from abc import abstractmethod
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter, UninitializedParameter
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed import (
|
||||
@@ -360,19 +360,6 @@ class ReplicatedLinear(LinearBase):
|
||||
self.register_parameter("bias", None)
|
||||
|
||||
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
|
||||
# If the weight on disk does not have a shape, give it one
|
||||
# (such scales for AutoFp8).
|
||||
# Special case for GGUF
|
||||
|
||||
is_gguf_weight = getattr(param, "is_gguf_weight", False)
|
||||
is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
|
||||
if is_gguf_weight_type:
|
||||
param.weight_type = loaded_weight.item()
|
||||
|
||||
# Materialize GGUF UninitializedParameter
|
||||
if is_gguf_weight and isinstance(param, UninitializedParameter):
|
||||
param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
|
||||
|
||||
if len(loaded_weight.shape) == 0:
|
||||
loaded_weight = loaded_weight.reshape(1)
|
||||
|
||||
@@ -536,20 +523,6 @@ class ColumnParallelLinear(LinearBase):
|
||||
# no need to narrow
|
||||
is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
|
||||
|
||||
# Special case for GGUF
|
||||
is_gguf_weight = getattr(param, "is_gguf_weight", False)
|
||||
is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
|
||||
if is_gguf_weight_type:
|
||||
param.weight_type = loaded_weight.item()
|
||||
|
||||
# Materialize GGUF UninitializedParameter
|
||||
if is_gguf_weight and isinstance(param, UninitializedParameter):
|
||||
final_shape = list(loaded_weight.shape)
|
||||
if output_dim is not None:
|
||||
assert final_shape[output_dim] % self.tp_size == 0
|
||||
final_shape[output_dim] = final_shape[output_dim] // self.tp_size
|
||||
param.materialize(final_shape, dtype=loaded_weight.dtype)
|
||||
|
||||
param_data = param.data
|
||||
if output_dim is not None and not is_sharded_weight:
|
||||
shard_size = param_data.shape[output_dim]
|
||||
@@ -693,37 +666,6 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
loaded_shard_id: tuple[int, ...] | int | None = None,
|
||||
):
|
||||
self.validate_shard_id(loaded_shard_id)
|
||||
# Special case for GGUF
|
||||
# initialize GGUF param after we know the quantize type
|
||||
is_gguf_weight = getattr(param, "is_gguf_weight", False)
|
||||
is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
|
||||
if isinstance(loaded_shard_id, tuple) and (
|
||||
is_gguf_weight or is_gguf_weight_type
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"Shard id with multiple indices is not supported for GGUF."
|
||||
)
|
||||
if is_gguf_weight_type:
|
||||
if loaded_shard_id is not None:
|
||||
param.data[loaded_shard_id].copy_(loaded_weight)
|
||||
param.shard_weight_type[loaded_shard_id] = loaded_weight.item()
|
||||
else:
|
||||
param.shard_weight_type = {
|
||||
i: loaded_weight.item() for i, _ in enumerate(self.output_sizes)
|
||||
}
|
||||
return
|
||||
|
||||
if is_gguf_weight:
|
||||
output_dim = getattr(param, "output_dim", None)
|
||||
shard_size = loaded_weight.size(output_dim) // self.tp_size
|
||||
start_idx = self.tp_rank * shard_size
|
||||
|
||||
if loaded_shard_id is not None:
|
||||
loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size)
|
||||
param.shard_id.append(loaded_shard_id)
|
||||
param.shard_id_map[loaded_shard_id] = len(param.data_container)
|
||||
param.data_container.append(loaded_weight)
|
||||
return
|
||||
|
||||
param_data = param.data
|
||||
output_dim = getattr(param, "output_dim", None)
|
||||
@@ -1186,30 +1128,6 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
loaded_shard_id: str | None = None,
|
||||
):
|
||||
self.validate_shard_id(loaded_shard_id)
|
||||
# Special case for GGUF
|
||||
# initialize GGUF param after we know the quantize type
|
||||
is_gguf_weight = getattr(param, "is_gguf_weight", False)
|
||||
is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
|
||||
if is_gguf_weight_type:
|
||||
idx_map = {"q": 0, "k": 1, "v": 2}
|
||||
if loaded_shard_id is not None:
|
||||
param.data[idx_map[loaded_shard_id]].copy_(loaded_weight)
|
||||
param.shard_weight_type[loaded_shard_id] = loaded_weight.item()
|
||||
else:
|
||||
param.shard_weight_type = {k: loaded_weight.item() for k in idx_map}
|
||||
return
|
||||
|
||||
if is_gguf_weight:
|
||||
output_dim = getattr(param, "output_dim", None)
|
||||
shard_size = loaded_weight.size(output_dim) // self.tp_size
|
||||
start_idx = self.tp_rank * shard_size
|
||||
|
||||
if loaded_shard_id is not None:
|
||||
loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size)
|
||||
param.shard_id.append(loaded_shard_id)
|
||||
param.shard_id_map[loaded_shard_id] = len(param.data_container)
|
||||
param.data_container.append(loaded_weight)
|
||||
return
|
||||
|
||||
param_data = param.data
|
||||
output_dim = getattr(param, "output_dim", None)
|
||||
@@ -1498,19 +1416,6 @@ class RowParallelLinear(LinearBase):
|
||||
# no need to narrow
|
||||
is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
|
||||
|
||||
# Special case for GGUF
|
||||
is_gguf_weight = getattr(param, "is_gguf_weight", False)
|
||||
is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
|
||||
if is_gguf_weight_type:
|
||||
param.weight_type = loaded_weight.item()
|
||||
|
||||
# Materialize GGUF UninitializedParameter
|
||||
if is_gguf_weight and isinstance(param, UninitializedParameter):
|
||||
weight_shape = list(loaded_weight.shape)
|
||||
if input_dim:
|
||||
weight_shape[input_dim] = weight_shape[input_dim] // self.tp_size
|
||||
param.materialize(tuple(weight_shape), dtype=loaded_weight.dtype)
|
||||
|
||||
param_data = param.data
|
||||
if input_dim is not None and not is_sharded_weight:
|
||||
shard_size = param_data.shape[input_dim]
|
||||
|
||||
@@ -18,7 +18,6 @@ QuantizationMethods = Literal[
|
||||
"modelopt_fp4",
|
||||
"modelopt_mxfp8",
|
||||
"modelopt_mixed",
|
||||
"gguf",
|
||||
"auto_gptq",
|
||||
"gptq",
|
||||
"gptq_marlin",
|
||||
@@ -125,7 +124,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
from .fbgemm_fp8 import FBGEMMFp8Config
|
||||
from .fp8 import Fp8Config
|
||||
from .fp_quant import FPQuantConfig
|
||||
from .gguf import GGUFConfig
|
||||
from .humming import HummingConfig
|
||||
from .inc import INCConfig
|
||||
from .modelopt import (
|
||||
@@ -148,7 +146,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
"modelopt_fp4": ModelOptNvFp4Config,
|
||||
"modelopt_mxfp8": ModelOptMxFp8Config,
|
||||
"modelopt_mixed": ModelOptMixedPrecisionConfig,
|
||||
"gguf": GGUFConfig,
|
||||
"auto_gptq": AutoGPTQConfig,
|
||||
"gptq": AutoGPTQConfig,
|
||||
"gptq_marlin": AutoGPTQConfig,
|
||||
|
||||
@@ -47,6 +47,13 @@ class QuantizeMethodBase(ABC):
|
||||
Expects create_weights to have been called before on the layer."""
|
||||
raise NotImplementedError
|
||||
|
||||
# Not required functions
|
||||
def tie_weights(self, layer: torch.nn.Module, *args, **kwargs):
|
||||
"""Tie layer's weights for the layer from another layer/tensors.
|
||||
|
||||
Expects create_weights to have been called before on the layer."""
|
||||
raise NotImplementedError
|
||||
|
||||
def process_weights_after_loading(self, layer: nn.Module) -> None:
|
||||
"""Process the weight after loading.
|
||||
|
||||
|
||||
@@ -1,690 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
|
||||
import gguf
|
||||
import torch
|
||||
from gguf import GGMLQuantizationType as WeightType
|
||||
from torch.nn.parameter import Parameter, UninitializedParameter
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEMethodBase,
|
||||
FusedMoEQuantConfig,
|
||||
MoEActivation,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
apply_moe_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
LinearMethodBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
UnquantizedEmbeddingMethod,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class GGUFConfig(QuantizationConfig):
|
||||
"""Config class for GGUF."""
|
||||
|
||||
def __init__(self, unquantized_modules: list[str] | None = None) -> None:
|
||||
super().__init__()
|
||||
self.unquantized_modules = unquantized_modules or []
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "GGUFConfig()"
|
||||
|
||||
def get_name(self) -> QuantizationMethods:
|
||||
return "gguf"
|
||||
|
||||
def get_supported_act_dtypes(self) -> list[torch.dtype]:
|
||||
# GGUF dequantization kernels use half precision (fp16) internally.
|
||||
# bfloat16 has precision issues on Blackwell devices.
|
||||
if current_platform.has_device_capability(100):
|
||||
logger.warning_once("GGUF has precision issues with bfloat16 on Blackwell.")
|
||||
return [torch.half, torch.float32]
|
||||
return [torch.half, torch.bfloat16, torch.float32]
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 60
|
||||
|
||||
@classmethod
|
||||
def get_config_filenames(cls) -> list[str]:
|
||||
return [] # no extra configs.
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "GGUFConfig":
|
||||
return cls()
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg: dict[str, Any], user_quant: str | None, hf_config=None
|
||||
) -> "QuantizationMethods | None":
|
||||
# When user explicitly specifies --quantization gguf, override
|
||||
# whatever quantization method is in the HF model config (e.g. fp8).
|
||||
if user_quant == "gguf":
|
||||
return "gguf"
|
||||
return None
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
if isinstance(layer, LinearBase):
|
||||
if is_layer_skipped_gguf(
|
||||
prefix, self.unquantized_modules, self.packed_modules_mapping
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
return GGUFLinearMethod(self)
|
||||
elif isinstance(layer, VocabParallelEmbedding):
|
||||
if is_layer_skipped_gguf(
|
||||
prefix, self.unquantized_modules, self.packed_modules_mapping
|
||||
):
|
||||
return UnquantizedEmbeddingMethod()
|
||||
return GGUFEmbeddingMethod(self)
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
# TODO: Select UnquantizedFusedMoEMethod on unquantized layers.
|
||||
return GGUFMoEMethod(self, layer.moe_config)
|
||||
return None
|
||||
|
||||
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
|
||||
"""
|
||||
Interface for models to update module names referenced in
|
||||
quantization configs in order to reflect the vllm model structure
|
||||
|
||||
Args:
|
||||
hf_to_vllm_mapper: maps from hf model structure (the assumed
|
||||
structure of the qconfig) to vllm model structure
|
||||
"""
|
||||
if self.unquantized_modules is not None:
|
||||
self.unquantized_modules = hf_to_vllm_mapper.apply_list(
|
||||
self.unquantized_modules
|
||||
)
|
||||
|
||||
|
||||
def is_layer_skipped_gguf(
|
||||
prefix: str,
|
||||
unquantized_modules: list[str],
|
||||
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
|
||||
):
|
||||
# Fused layers like gate_up_proj or qkv_proj will not be fused
|
||||
# in the safetensors checkpoint. So, we convert the name
|
||||
# from the fused version to unfused + check to make sure that
|
||||
# each shard of the fused layer has the same scheme.
|
||||
proj_name = prefix.split(".")[-1]
|
||||
if proj_name in fused_mapping:
|
||||
shard_prefixes = [
|
||||
prefix.replace(proj_name, shard_proj_name)
|
||||
for shard_proj_name in fused_mapping[proj_name]
|
||||
]
|
||||
|
||||
is_skipped = None
|
||||
for shard_prefix in shard_prefixes:
|
||||
is_shard_skipped = any(
|
||||
shard_prefix in module_name for module_name in unquantized_modules
|
||||
)
|
||||
|
||||
if is_skipped is None:
|
||||
is_skipped = is_shard_skipped
|
||||
elif is_shard_skipped != is_skipped:
|
||||
raise ValueError(
|
||||
f"Detected some but not all shards of {prefix} "
|
||||
"are quantized. All shards of fused layers "
|
||||
"to have the same precision."
|
||||
)
|
||||
else:
|
||||
is_skipped = any(module_name in prefix for module_name in unquantized_modules)
|
||||
|
||||
assert is_skipped is not None
|
||||
return is_skipped
|
||||
|
||||
|
||||
UNQUANTIZED_TYPES = {WeightType.F32, WeightType.F16, WeightType.BF16}
|
||||
STANDARD_QUANT_TYPES = {
|
||||
WeightType.Q4_0,
|
||||
WeightType.Q4_1,
|
||||
WeightType.Q5_0,
|
||||
WeightType.Q5_1,
|
||||
WeightType.Q8_0,
|
||||
WeightType.Q8_1,
|
||||
}
|
||||
KQUANT_TYPES = {
|
||||
WeightType.Q2_K,
|
||||
WeightType.Q3_K,
|
||||
WeightType.Q4_K,
|
||||
WeightType.Q5_K,
|
||||
WeightType.Q6_K,
|
||||
}
|
||||
IMATRIX_QUANT_TYPES = {
|
||||
WeightType.IQ1_M,
|
||||
WeightType.IQ1_S,
|
||||
WeightType.IQ2_XXS,
|
||||
WeightType.IQ2_XS,
|
||||
WeightType.IQ2_S,
|
||||
WeightType.IQ3_XXS,
|
||||
WeightType.IQ3_S,
|
||||
WeightType.IQ4_XS,
|
||||
WeightType.IQ4_NL,
|
||||
}
|
||||
# TODO(Isotr0py): Currently, we don't have MMQ kernel for I-Matrix quantization.
|
||||
# Consolidate DEQUANT_TYPES, MMVQ_QUANT_TYPES and MMQ_QUANT_TYPES after we add
|
||||
# MMQ kernel for I-Matrix quantization.
|
||||
DEQUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES | IMATRIX_QUANT_TYPES
|
||||
MMVQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES | IMATRIX_QUANT_TYPES
|
||||
MMQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES
|
||||
|
||||
|
||||
def _fused_mul_mat_gguf(
|
||||
x: torch.Tensor, qweight: torch.Tensor, qweight_type: int
|
||||
) -> torch.Tensor:
|
||||
if qweight_type in IMATRIX_QUANT_TYPES:
|
||||
mmvq_safe = 8 if qweight.shape[0] > 5120 else 16
|
||||
else:
|
||||
mmvq_safe = 2 if qweight.shape[0] > 5120 else 6
|
||||
# HACK: when doing chunked prefill we don't generate output tokens
|
||||
# so input to logits generator is empty which causes invalid parameter
|
||||
if x.shape[0] == 0:
|
||||
return torch.empty(x.shape[0], qweight.shape[0], dtype=x.dtype, device=x.device)
|
||||
# there is no need to call any kernel for fp16/bf16
|
||||
if qweight_type in UNQUANTIZED_TYPES:
|
||||
return x @ qweight.T
|
||||
# enable MMVQ in contiguous batching with batch_size=1
|
||||
if x.shape[0] <= mmvq_safe and qweight_type in MMVQ_QUANT_TYPES:
|
||||
y = ops.ggml_mul_mat_vec_a8(qweight, x, qweight_type, qweight.shape[0])
|
||||
# Use MMQ Kernel if it's available (standard + k-quants)
|
||||
elif qweight_type in MMQ_QUANT_TYPES:
|
||||
y = ops.ggml_mul_mat_a8(qweight, x, qweight_type, qweight.shape[0])
|
||||
# If there is no available MMQ kernel, fallback to dequantize
|
||||
elif qweight_type in DEQUANT_TYPES:
|
||||
block_size, type_size = gguf.GGML_QUANT_SIZES[qweight_type]
|
||||
shape = (qweight.shape[0], qweight.shape[1] // type_size * block_size)
|
||||
weight = ops.ggml_dequantize(qweight, qweight_type, *shape, x.dtype)
|
||||
y = x @ weight.T
|
||||
else:
|
||||
# Raise an error if the quantization type is not supported.
|
||||
# Might be useful if llama.cpp adds a new quantization type.
|
||||
# Wrap to GGMLQuantizationType IntEnum to make sure it's a valid type.
|
||||
qweight_type = WeightType(qweight_type)
|
||||
raise NotImplementedError(f"Unsupported GGUF quantization type: {qweight_type}")
|
||||
return y
|
||||
|
||||
|
||||
def _fused_mul_mat_gguf_fake(
|
||||
x: torch.Tensor,
|
||||
qweight: torch.Tensor,
|
||||
qweight_type: int,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(x.shape[0], qweight.shape[0], dtype=x.dtype, device=x.device)
|
||||
|
||||
|
||||
try:
|
||||
direct_register_custom_op(
|
||||
op_name="_fused_mul_mat_gguf",
|
||||
op_func=_fused_mul_mat_gguf,
|
||||
fake_impl=_fused_mul_mat_gguf_fake,
|
||||
)
|
||||
fused_mul_mat_gguf = torch.ops.vllm._fused_mul_mat_gguf
|
||||
|
||||
except AttributeError as error:
|
||||
raise error
|
||||
|
||||
|
||||
def _fused_moe_gguf(
|
||||
x: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
qweight_type: int,
|
||||
qweight_type2: int,
|
||||
activation: str,
|
||||
) -> torch.Tensor:
|
||||
activation_enum = MoEActivation.from_str(activation)
|
||||
|
||||
def act(x: torch.Tensor):
|
||||
d = x.shape[-1] // 2
|
||||
output_shape = x.shape[:-1] + (d,)
|
||||
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
|
||||
apply_moe_activation(activation_enum, out, x)
|
||||
return out
|
||||
|
||||
# lazy import to avoid triggering triton import in CPU backend
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import moe_align_block_size
|
||||
|
||||
out_hidden_states = torch.empty_like(x)
|
||||
# unless we decent expert reuse we are better off running moe_vec kernel
|
||||
if (
|
||||
qweight_type2 in MMQ_QUANT_TYPES
|
||||
and qweight_type in MMQ_QUANT_TYPES
|
||||
and x.shape[0] > 64
|
||||
):
|
||||
num_tokens, _ = x.shape
|
||||
E, N, _ = w1.shape
|
||||
top_k = topk_ids.shape[1]
|
||||
BLOCK_SIZE = ops.ggml_moe_get_block_size(qweight_type)
|
||||
|
||||
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
|
||||
topk_ids, BLOCK_SIZE, E
|
||||
)
|
||||
out = ops.ggml_moe_a8(
|
||||
x,
|
||||
w1,
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_padded,
|
||||
qweight_type,
|
||||
N,
|
||||
top_k,
|
||||
num_tokens,
|
||||
)
|
||||
out = act(out)
|
||||
out = ops.ggml_moe_a8(
|
||||
out,
|
||||
w2,
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_padded,
|
||||
qweight_type2,
|
||||
w2.shape[1],
|
||||
1,
|
||||
num_tokens * top_k,
|
||||
)
|
||||
out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_(
|
||||
topk_weights.view(num_tokens, top_k, 1)
|
||||
)
|
||||
ops.moe_sum(out, out_hidden_states)
|
||||
elif qweight_type2 in MMVQ_QUANT_TYPES and qweight_type in MMVQ_QUANT_TYPES:
|
||||
num_tokens, _ = x.shape
|
||||
E, N, _ = w1.shape
|
||||
top_k = topk_ids.shape[1]
|
||||
|
||||
out = ops.ggml_moe_a8_vec(x, w1, topk_ids, top_k, qweight_type, N, num_tokens)
|
||||
out = act(out)
|
||||
|
||||
out = ops.ggml_moe_a8_vec(
|
||||
out, w2, topk_ids, 1, qweight_type2, w2.shape[1], num_tokens * top_k
|
||||
)
|
||||
out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_(
|
||||
topk_weights.view(num_tokens, top_k, 1)
|
||||
)
|
||||
ops.moe_sum(out, out_hidden_states)
|
||||
else:
|
||||
logger.warning_once(
|
||||
"There is no support for fast MoE kernel "
|
||||
"for current quantization method. "
|
||||
"Falling back to slow implementation. "
|
||||
)
|
||||
for tok, (w, idx) in enumerate(zip(topk_weights, topk_ids)):
|
||||
inp = x[tok].reshape((1,) + x.shape[1:])
|
||||
current_hidden_state = None
|
||||
for ww, ii in zip(w, idx):
|
||||
expert_up = w1[ii]
|
||||
|
||||
out = fused_mul_mat_gguf(inp, expert_up, qweight_type)
|
||||
out = act(out)
|
||||
|
||||
expert_down = w2[ii]
|
||||
current_state = fused_mul_mat_gguf(
|
||||
out, expert_down, qweight_type2
|
||||
).mul_(ww)
|
||||
if current_hidden_state is None:
|
||||
current_hidden_state = current_state
|
||||
else:
|
||||
current_hidden_state.add_(current_state)
|
||||
out_hidden_states[tok] = current_hidden_state
|
||||
return out_hidden_states
|
||||
|
||||
|
||||
def _fused_moe_gguf_fake(
|
||||
x: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
qweight_type: int,
|
||||
qweight_type2: int,
|
||||
activation: str,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(x)
|
||||
|
||||
|
||||
try:
|
||||
direct_register_custom_op(
|
||||
op_name="_fused_moe_gguf",
|
||||
op_func=_fused_moe_gguf,
|
||||
fake_impl=_fused_moe_gguf_fake,
|
||||
)
|
||||
fused_moe_gguf = torch.ops.vllm._fused_moe_gguf
|
||||
|
||||
except AttributeError as error:
|
||||
raise error
|
||||
|
||||
|
||||
def _apply_gguf_embedding(
|
||||
x: torch.Tensor,
|
||||
qweight: torch.Tensor,
|
||||
qweight_type: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
if qweight_type in UNQUANTIZED_TYPES:
|
||||
return torch.embedding(qweight, x)
|
||||
elif qweight_type in DEQUANT_TYPES:
|
||||
block_size, type_size = gguf.GGML_QUANT_SIZES[qweight_type]
|
||||
x_flat = x.flatten()
|
||||
assert hidden_size == qweight.shape[1] // type_size * block_size
|
||||
quant = torch.index_select(qweight, dim=0, index=x_flat)
|
||||
dequant = ops.ggml_dequantize(
|
||||
quant, qweight_type, hidden_size, x_flat.shape[0], dtype
|
||||
)
|
||||
return dequant.view(*x.shape, hidden_size)
|
||||
else:
|
||||
qweight_type = WeightType(qweight_type)
|
||||
raise NotImplementedError(f"Unsupported GGUF quantization type: {qweight_type}")
|
||||
|
||||
|
||||
def _apply_gguf_embedding_fake(
|
||||
x: torch.Tensor,
|
||||
qweight: torch.Tensor,
|
||||
qweight_type: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(x.shape[0], hidden_size, dtype=dtype, device=x.device)
|
||||
|
||||
|
||||
try:
|
||||
direct_register_custom_op(
|
||||
op_name="_apply_gguf_embedding",
|
||||
op_func=_apply_gguf_embedding,
|
||||
fake_impl=_apply_gguf_embedding_fake,
|
||||
)
|
||||
apply_gguf_embedding = torch.ops.vllm._apply_gguf_embedding
|
||||
|
||||
except AttributeError as error:
|
||||
raise error
|
||||
|
||||
|
||||
class GGUFLinearMethod(LinearMethodBase):
|
||||
"""Linear method for GGUF.
|
||||
|
||||
Args:
|
||||
quant_config: The GGUF quantization config.
|
||||
"""
|
||||
|
||||
def __init__(self, quant_config: GGUFConfig):
|
||||
self.quant_config = quant_config
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
self.params_dtype = params_dtype
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
|
||||
tensor_shape = (output_size_per_partition, input_size_per_partition)
|
||||
qweight = GGUFUninitializedParameter(requires_grad=False)
|
||||
set_weight_attrs(
|
||||
qweight,
|
||||
{
|
||||
"input_dim": 1,
|
||||
"output_dim": 0,
|
||||
"tensor_shape": tensor_shape,
|
||||
"is_gguf_weight": True,
|
||||
"data_container": [],
|
||||
"shard_id": [],
|
||||
"shard_id_map": {},
|
||||
},
|
||||
)
|
||||
set_weight_attrs(qweight, extra_weight_attrs)
|
||||
layer.register_parameter("qweight", qweight)
|
||||
|
||||
qweight_type = Parameter(
|
||||
torch.empty(len(output_partition_sizes), dtype=torch.uint8),
|
||||
requires_grad=False,
|
||||
)
|
||||
set_weight_attrs(
|
||||
qweight_type,
|
||||
{
|
||||
"is_gguf_weight_type": True,
|
||||
"weight_type": 0,
|
||||
"shard_weight_type": {},
|
||||
"ignore_warning": True,
|
||||
},
|
||||
)
|
||||
set_weight_attrs(qweight_type, extra_weight_attrs)
|
||||
layer.register_parameter("qweight_type", qweight_type)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
qweight_type = layer.qweight_type.weight_type
|
||||
if not (qweight_type in UNQUANTIZED_TYPES or qweight_type in DEQUANT_TYPES):
|
||||
qweight_type = WeightType(qweight_type)
|
||||
raise ValueError(
|
||||
f"Unsupported GGUF quantization type {qweight_type} in layer {layer}."
|
||||
)
|
||||
# For MergedColumnParallelLinear and QKVParallelLinear, we need to
|
||||
# materialize the padded weight parameter for CUDA Graph compatibility.
|
||||
self._create_padded_weight_param(layer)
|
||||
|
||||
def _create_padded_weight_param(self, layer: torch.nn.Module):
|
||||
"""Create padded weight parameter for GGUF MergedLinear layer."""
|
||||
qweight = layer.qweight
|
||||
shard_id_map = qweight.shard_id_map
|
||||
shard_id = qweight.shard_id
|
||||
if len(data_container := qweight.data_container) > 1:
|
||||
dtype = {data.dtype for data in data_container}
|
||||
assert len(dtype) == 1, ValueError(
|
||||
f"Data container has mixed dtypes: {dtype}"
|
||||
)
|
||||
dtype = next(iter(dtype))
|
||||
# concat dim0 and pad dim1
|
||||
padded_side = max(x.size(1) for x in data_container)
|
||||
concat_side = sum(x.size(0) for x in data_container)
|
||||
# Pad the quantized weights to dense tensor, and create a map
|
||||
# with the location of each shard in the padded tensor.
|
||||
padded_data = torch.zeros(
|
||||
(concat_side, padded_side), dtype=dtype, device=qweight.device
|
||||
)
|
||||
# (dim0_start, dim0_end, dim1_size)
|
||||
shard_offset_map = dict[str, tuple[int, int, int]]()
|
||||
for idx in shard_id:
|
||||
id_in_container = shard_id_map[idx]
|
||||
start = sum(x.size(0) for x in data_container[:id_in_container])
|
||||
end = start + data_container[id_in_container].size(0)
|
||||
size = data_container[id_in_container].size(1)
|
||||
padded_data[start:end, :size] = data_container[id_in_container]
|
||||
shard_offset_map[idx] = (start, end, size)
|
||||
qweight.data_container.clear()
|
||||
padded_param = Parameter(padded_data, requires_grad=False)
|
||||
set_weight_attrs(padded_param, vars(qweight))
|
||||
set_weight_attrs(padded_param, {"shard_offset_map": shard_offset_map})
|
||||
layer.register_parameter("qweight", padded_param)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
shard_id = layer.qweight.shard_id
|
||||
|
||||
if shard_id:
|
||||
# dequantize shard weights respectively
|
||||
shard_id = ["q", "k", "v"] if "q" in shard_id else shard_id
|
||||
qweight = layer.qweight
|
||||
result = []
|
||||
for idx in shard_id:
|
||||
start, end, offset = layer.qweight.shard_offset_map[idx]
|
||||
qweight_type = layer.qweight_type.shard_weight_type[idx]
|
||||
result.append(
|
||||
fused_mul_mat_gguf(
|
||||
x, qweight[start:end, :offset].contiguous(), qweight_type
|
||||
)
|
||||
)
|
||||
out = torch.cat(result, axis=1)
|
||||
else:
|
||||
qweight = layer.qweight
|
||||
qweight_type = layer.qweight_type.weight_type
|
||||
out = fused_mul_mat_gguf(x, qweight, qweight_type)
|
||||
if bias is not None:
|
||||
out.add_(bias)
|
||||
return out
|
||||
|
||||
|
||||
class GGUFMoEMethod(FusedMoEMethodBase):
|
||||
"""MoE method for GGUF.
|
||||
|
||||
Args:
|
||||
quant_config: The GGUF quantization config.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
quant_config: GGUFConfig,
|
||||
moe: FusedMoEConfig,
|
||||
):
|
||||
super().__init__(moe)
|
||||
self.quant_config = quant_config
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
tensor_shape = (num_experts, 2 * intermediate_size_per_partition, hidden_size)
|
||||
# gate up proj
|
||||
w13_qweight = GGUFUninitializedParameter(requires_grad=False)
|
||||
set_weight_attrs(
|
||||
w13_qweight,
|
||||
{
|
||||
"input_dim": 1,
|
||||
"output_dim": 0,
|
||||
"tensor_shape": tensor_shape,
|
||||
"is_gguf_weight": True,
|
||||
"data_container": [],
|
||||
},
|
||||
)
|
||||
set_weight_attrs(w13_qweight, extra_weight_attrs)
|
||||
layer.register_parameter("w13_qweight", w13_qweight)
|
||||
|
||||
w13_qweight_type = Parameter(
|
||||
torch.empty(1, dtype=torch.uint8), requires_grad=False
|
||||
)
|
||||
set_weight_attrs(
|
||||
w13_qweight_type,
|
||||
{"is_gguf_weight_type": True, "weight_type": 0, "ignore_warning": True},
|
||||
)
|
||||
set_weight_attrs(w13_qweight_type, extra_weight_attrs)
|
||||
layer.register_parameter("w13_qweight_type", w13_qweight_type)
|
||||
|
||||
tensor_shape = (num_experts, intermediate_size_per_partition, hidden_size)
|
||||
# gate down proj
|
||||
w2_qweight = GGUFUninitializedParameter(requires_grad=False)
|
||||
set_weight_attrs(
|
||||
w2_qweight,
|
||||
{
|
||||
"input_dim": 1,
|
||||
"output_dim": 0,
|
||||
"tensor_shape": tensor_shape,
|
||||
"is_gguf_weight": True,
|
||||
"data_container": [],
|
||||
},
|
||||
)
|
||||
set_weight_attrs(w2_qweight, extra_weight_attrs)
|
||||
layer.register_parameter("w2_qweight", w2_qweight)
|
||||
|
||||
w2_qweight_type = Parameter(
|
||||
torch.empty(1, dtype=torch.uint8), requires_grad=False
|
||||
)
|
||||
set_weight_attrs(
|
||||
w2_qweight_type,
|
||||
{"is_gguf_weight_type": True, "weight_type": 0, "ignore_warning": True},
|
||||
)
|
||||
|
||||
set_weight_attrs(w2_qweight_type, extra_weight_attrs)
|
||||
layer.register_parameter("w2_qweight_type", w2_qweight_type)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: RoutedExperts
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
return None
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
if layer.apply_router_weight_on_input:
|
||||
raise NotImplementedError(
|
||||
"Apply router weight on input is not supported for"
|
||||
"fused GGUF MoE method."
|
||||
)
|
||||
|
||||
return fused_moe_gguf(
|
||||
x,
|
||||
layer.w13_qweight,
|
||||
layer.w2_qweight,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
layer.w13_qweight_type.weight_type,
|
||||
layer.w2_qweight_type.weight_type,
|
||||
layer.activation.value,
|
||||
)
|
||||
|
||||
|
||||
class GGUFEmbeddingMethod(GGUFLinearMethod):
|
||||
"""Embedding method for GGUF.
|
||||
|
||||
Args:
|
||||
quant_config: The GGUF quantization config.
|
||||
"""
|
||||
|
||||
def embedding(self, layer: torch.nn.Module, x: torch.Tensor) -> torch.Tensor:
|
||||
qweight = layer.qweight
|
||||
qweight_type = layer.qweight_type.weight_type
|
||||
hidden_size = qweight.tensor_shape[1]
|
||||
|
||||
return apply_gguf_embedding(
|
||||
x, qweight, qweight_type, hidden_size, dtype=self.params_dtype
|
||||
)
|
||||
|
||||
|
||||
class GGUFUninitializedParameter(UninitializedParameter):
|
||||
cls_to_become = Parameter
|
||||
data_container: list[torch.Tensor]
|
||||
@@ -6,7 +6,7 @@ from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch.nn.parameter import Parameter, UninitializedParameter
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed import (
|
||||
@@ -77,6 +77,12 @@ class UnquantizedEmbeddingMethod(QuantizeMethodBase):
|
||||
def embedding(self, layer: torch.nn.Module, input_: torch.Tensor) -> torch.Tensor:
|
||||
return F.embedding(input_, layer.weight)
|
||||
|
||||
def tie_weights(
|
||||
self, layer: torch.nn.Module, embed_tokens: "VocabParallelEmbedding"
|
||||
):
|
||||
layer.weight = embed_tokens.weight
|
||||
return layer
|
||||
|
||||
|
||||
def pad_vocab_size(vocab_size: int, pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int:
|
||||
"""Pad the vocab size to the given value."""
|
||||
@@ -425,17 +431,6 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
output_dim = getattr(param, "output_dim", None)
|
||||
packed_dim = getattr(param, "packed_dim", None)
|
||||
|
||||
# If the parameter is a gguf weight, then load it directly.
|
||||
if getattr(param, "is_gguf_weight_type", None):
|
||||
param.data.copy_(loaded_weight)
|
||||
param.weight_type = loaded_weight.item()
|
||||
return
|
||||
elif isinstance(param, UninitializedParameter):
|
||||
shape = list(loaded_weight.shape)
|
||||
if output_dim is not None:
|
||||
shape[output_dim] = self.num_embeddings_per_partition
|
||||
param.materialize(tuple(shape), dtype=loaded_weight.dtype)
|
||||
|
||||
# If parameter does not have output dim, then it should
|
||||
# be copied onto all gpus (e.g. g_idx for act_order gptq).
|
||||
if output_dim is None:
|
||||
@@ -562,12 +557,7 @@ class ParallelLMHead(VocabParallelEmbedding):
|
||||
|
||||
def tie_weights(self, embed_tokens: VocabParallelEmbedding):
|
||||
"""Tie the weights with word embeddings."""
|
||||
# GGUF quantized embed_tokens.
|
||||
if self.quant_config and self.quant_config.get_name() == "gguf":
|
||||
return embed_tokens
|
||||
else:
|
||||
self.weight = embed_tokens.weight
|
||||
return self
|
||||
return self.quant_method.tie_weights(self, embed_tokens)
|
||||
|
||||
def forward(self, input_):
|
||||
del input_
|
||||
|
||||
@@ -12,7 +12,6 @@ from vllm.model_executor.model_loader.base_loader import BaseModelLoader
|
||||
from vllm.model_executor.model_loader.bitsandbytes_loader import BitsAndBytesModelLoader
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
|
||||
from vllm.model_executor.model_loader.gguf_loader import GGUFModelLoader
|
||||
from vllm.model_executor.model_loader.modelexpress_loader import (
|
||||
ModelExpressModelLoader,
|
||||
)
|
||||
@@ -37,7 +36,6 @@ LoadFormats = Literal[
|
||||
"bitsandbytes",
|
||||
"dummy",
|
||||
"fastsafetensors",
|
||||
"gguf",
|
||||
"instanttensor",
|
||||
"mistral",
|
||||
"modelexpress",
|
||||
@@ -55,7 +53,6 @@ _LOAD_FORMAT_TO_MODEL_LOADER: dict[str, type[BaseModelLoader]] = {
|
||||
"bitsandbytes": BitsAndBytesModelLoader,
|
||||
"dummy": DummyModelLoader,
|
||||
"fastsafetensors": DefaultModelLoader,
|
||||
"gguf": GGUFModelLoader,
|
||||
"instanttensor": DefaultModelLoader,
|
||||
"mistral": DefaultModelLoader,
|
||||
"modelexpress": ModelExpressModelLoader,
|
||||
@@ -154,7 +151,6 @@ __all__ = [
|
||||
"register_model_loader",
|
||||
"BaseModelLoader",
|
||||
"BitsAndBytesModelLoader",
|
||||
"GGUFModelLoader",
|
||||
"ModelExpressModelLoader",
|
||||
"DefaultModelLoader",
|
||||
"DummyModelLoader",
|
||||
|
||||
@@ -1,453 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
from collections.abc import Generator
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import gguf
|
||||
import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import AutoModelForCausalLM, AutoModelForImageTextToText
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
|
||||
from vllm.model_executor.model_loader.utils import (
|
||||
initialize_model,
|
||||
process_weights_after_loading,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
download_gguf,
|
||||
get_gguf_extra_tensor_names,
|
||||
get_gguf_weight_type_map,
|
||||
gguf_quant_weights_iterator,
|
||||
gguf_quant_weights_iterator_multi,
|
||||
)
|
||||
from vllm.transformers_utils.gguf_utils import detect_gguf_multimodal
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.gguf import GGUFConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class GGUFModelLoader(BaseModelLoader):
|
||||
"""
|
||||
Model loader that can load GGUF files. This is useful for loading models
|
||||
that are quantized with GGUF and saved in the GGUF format. This loader
|
||||
supports loading both full models and sharded models.
|
||||
"""
|
||||
|
||||
def __init__(self, load_config: LoadConfig):
|
||||
super().__init__(load_config)
|
||||
if load_config.model_loader_extra_config:
|
||||
raise ValueError(
|
||||
f"Model loader extra config is not supported for "
|
||||
f"load format {load_config.load_format}"
|
||||
)
|
||||
|
||||
def _prepare_weights(self, model_config: ModelConfig):
|
||||
model_name_or_path = model_config.model
|
||||
if os.path.isfile(model_name_or_path):
|
||||
return model_name_or_path
|
||||
# repo id/filename.gguf
|
||||
if "/" in model_name_or_path and model_name_or_path.endswith(".gguf"):
|
||||
repo_id, filename = model_name_or_path.rsplit("/", 1)
|
||||
return hf_api().hf_hub_download(
|
||||
repo_id=repo_id,
|
||||
filename=filename,
|
||||
revision=model_config.revision,
|
||||
cache_dir=self.load_config.download_dir,
|
||||
)
|
||||
# repo_id:quant_type
|
||||
elif "/" in model_name_or_path and ":" in model_name_or_path:
|
||||
repo_id, quant_type = model_name_or_path.rsplit(":", 1)
|
||||
return download_gguf(
|
||||
repo_id,
|
||||
quant_type,
|
||||
cache_dir=self.load_config.download_dir,
|
||||
revision=model_config.revision,
|
||||
ignore_patterns=self.load_config.ignore_patterns,
|
||||
)
|
||||
|
||||
raise ValueError(
|
||||
f"Unrecognised GGUF reference: {model_name_or_path} "
|
||||
"(expected local file, <repo_id>/<filename>.gguf, "
|
||||
"or <repo_id>:<quant_type>)"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_all_gguf_files(model_path: str) -> list[str]:
|
||||
"""Discover all GGUF shard files from a single shard path.
|
||||
|
||||
Supports variable-width shard indices by dynamically detecting
|
||||
the padding from the original filename.
|
||||
E.g. ``*-00001-of-00005.gguf`` → all 5 shards,
|
||||
``*-01-of-15.gguf`` → all 15 shards.
|
||||
"""
|
||||
match = re.search(r"-(\d+)-of-(\d+)\.gguf$", model_path)
|
||||
if not match:
|
||||
return [model_path]
|
||||
total = int(match.group(2))
|
||||
num_digits = len(match.group(1))
|
||||
prefix = model_path[: match.start(1)]
|
||||
suffix = model_path[match.end(2) :]
|
||||
files = []
|
||||
for i in range(1, total + 1):
|
||||
shard_path = f"{prefix}{i:0{num_digits}d}-of-{total:0{num_digits}d}{suffix}"
|
||||
if os.path.isfile(shard_path):
|
||||
files.append(shard_path)
|
||||
if files:
|
||||
logger.info("Discovered %d GGUF shard files", len(files))
|
||||
return files if files else [model_path]
|
||||
|
||||
def _get_gguf_weights_map(self, model_config: ModelConfig):
|
||||
"""
|
||||
GGUF uses this naming convention for their tensors from HF checkpoint:
|
||||
`blk.N.BB.weight` and `blk.N.BB.bias`
|
||||
where N signifies the block number of a layer, and BB signifies the
|
||||
attention/mlp layer components.
|
||||
See "Standardized tensor names" in
|
||||
https://github.com/ggerganov/ggml/blob/master/docs/gguf.md for details.
|
||||
"""
|
||||
config = model_config.hf_config
|
||||
# Get text config to handle both nested (multimodal) and flat
|
||||
# (text-only) config structures. For multimodal models like
|
||||
# Gemma3Config, this returns config.text_config. For text-only
|
||||
# models, this returns config itself.
|
||||
text_config = config.get_text_config()
|
||||
model_type = config.model_type
|
||||
is_multimodal = (
|
||||
hasattr(config, "vision_config") and config.vision_config is not None
|
||||
)
|
||||
gguf_to_hf_name_map = {}
|
||||
sideload_params: list[re.Pattern] = []
|
||||
# hack: ggufs have a different name than transformers
|
||||
if model_type == "cohere":
|
||||
model_type = "command-r"
|
||||
if model_type == "gemma3_text":
|
||||
# Gemma3 models use "gemma3_text" in HuggingFace but
|
||||
# "gemma3" in GGUF architecture naming
|
||||
model_type = "gemma3"
|
||||
if model_type in ("deepseek_v3", "deepseek_v2"):
|
||||
model_type = "deepseek2"
|
||||
# GGUF layer map assumes that we will have a merged expert weights
|
||||
# so we need to map them manually
|
||||
for idx in range(config.num_hidden_layers):
|
||||
gguf_to_hf_name_map[f"blk.{idx}.exp_probs_b.bias"] = (
|
||||
f"model.layers.{idx}.mlp.gate.e_score_correction_bias"
|
||||
)
|
||||
gguf_to_hf_name_map[f"blk.{idx}.ffn_down_exps.weight"] = (
|
||||
f"model.layers.{idx}.mlp.experts.0.down_proj.weight"
|
||||
)
|
||||
gguf_to_hf_name_map[f"blk.{idx}.ffn_gate_exps.weight"] = (
|
||||
f"model.layers.{idx}.mlp.experts.0.gate_proj.weight"
|
||||
)
|
||||
gguf_to_hf_name_map[f"blk.{idx}.ffn_up_exps.weight"] = (
|
||||
f"model.layers.{idx}.mlp.experts.0.up_proj.weight"
|
||||
)
|
||||
sideload_params.append(
|
||||
re.compile(
|
||||
f"model\\.layers\\.{idx}"
|
||||
r"\.mlp\.experts\.[0-9]+\.(gate|up|down)_proj\.weight"
|
||||
)
|
||||
)
|
||||
if model_type in ("qwen2_moe", "qwen3_moe"):
|
||||
model_type = model_type.replace("_", "")
|
||||
# GGUF layer map assumes that we will have a merged expert weights
|
||||
# so we need to map them manually
|
||||
for idx in range(config.num_hidden_layers):
|
||||
gguf_to_hf_name_map[f"blk.{idx}.ffn_down_exps.weight"] = (
|
||||
f"model.layers.{idx}.mlp.experts.0.down_proj.weight"
|
||||
)
|
||||
gguf_to_hf_name_map[f"blk.{idx}.ffn_gate_exps.weight"] = (
|
||||
f"model.layers.{idx}.mlp.experts.0.gate_proj.weight"
|
||||
)
|
||||
gguf_to_hf_name_map[f"blk.{idx}.ffn_up_exps.weight"] = (
|
||||
f"model.layers.{idx}.mlp.experts.0.up_proj.weight"
|
||||
)
|
||||
sideload_params.append(
|
||||
re.compile(
|
||||
f"model\\.layers\\.{idx}"
|
||||
r"\.mlp\.experts\.[0-9]+\.(gate|up|down)_proj\.weight"
|
||||
)
|
||||
)
|
||||
if model_type == "minimax_m2":
|
||||
model_type = "minimax-m2"
|
||||
# GGUF layer map assumes merged expert weights
|
||||
# map them manually like deepseek2
|
||||
for idx in range(config.num_hidden_layers):
|
||||
gguf_to_hf_name_map[f"blk.{idx}.exp_probs_b.bias"] = (
|
||||
f"model.layers.{idx}.block_sparse_moe.e_score_correction_bias"
|
||||
)
|
||||
gguf_to_hf_name_map[f"blk.{idx}.ffn_down_exps.weight"] = (
|
||||
f"model.layers.{idx}.block_sparse_moe.experts.0.w2.weight"
|
||||
)
|
||||
gguf_to_hf_name_map[f"blk.{idx}.ffn_gate_exps.weight"] = (
|
||||
f"model.layers.{idx}.block_sparse_moe.experts.0.w1.weight"
|
||||
)
|
||||
gguf_to_hf_name_map[f"blk.{idx}.ffn_up_exps.weight"] = (
|
||||
f"model.layers.{idx}.block_sparse_moe.experts.0.w3.weight"
|
||||
)
|
||||
sideload_params.append(
|
||||
re.compile(
|
||||
f"model\\.layers\\.{idx}"
|
||||
r"\.block_sparse_moe\.experts\.(gate_up_proj|down_proj)"
|
||||
)
|
||||
)
|
||||
|
||||
arch = None
|
||||
for key, value in gguf.MODEL_ARCH_NAMES.items():
|
||||
if value == model_type:
|
||||
arch = key
|
||||
break
|
||||
if arch is None:
|
||||
raise RuntimeError(f"Unknown gguf model_type: {model_type}")
|
||||
text_num_layers = text_config.num_hidden_layers
|
||||
text_name_map = gguf.get_tensor_name_map(arch, text_num_layers)
|
||||
|
||||
if is_multimodal:
|
||||
mm_proj_arch = gguf.MODEL_ARCH.MMPROJ
|
||||
vision_num_layers = config.vision_config.num_hidden_layers
|
||||
vision_name_map = gguf.get_tensor_name_map(mm_proj_arch, vision_num_layers)
|
||||
else:
|
||||
vision_name_map = None
|
||||
|
||||
# Create dummy model to extract parameter names
|
||||
# For multimodal: use AutoModelForImageTextToText to get
|
||||
# language + vision + projector params
|
||||
# For text-only: use AutoModelForCausalLM to get language model params
|
||||
auto_cls = (
|
||||
AutoModelForImageTextToText if is_multimodal else AutoModelForCausalLM
|
||||
)
|
||||
with torch.device("meta"):
|
||||
dummy_model = auto_cls.from_config(
|
||||
config, trust_remote_code=model_config.trust_remote_code
|
||||
)
|
||||
|
||||
state_dict = dummy_model.state_dict()
|
||||
if hf_checkpoint_map := getattr(
|
||||
dummy_model, "_checkpoint_conversion_mapping", None
|
||||
):
|
||||
|
||||
def revert_hf_rename(name: str) -> str:
|
||||
for original_name, hf_name in hf_checkpoint_map.items():
|
||||
if hf_name in name:
|
||||
name = name.replace(hf_name, original_name).lstrip("^")
|
||||
return name
|
||||
|
||||
state_dict = {
|
||||
revert_hf_rename(name): tensor for name, tensor in state_dict.items()
|
||||
}
|
||||
|
||||
if model_type == "minimax-m2" and not hf_checkpoint_map:
|
||||
# Reverse HF convention: mlp -> block_sparse_moe
|
||||
state_dict = {
|
||||
name.replace(".mlp.", ".block_sparse_moe."): tensor
|
||||
for name, tensor in state_dict.items()
|
||||
}
|
||||
|
||||
def find_hf_name_in_tensor_map(hf_name: str) -> str | None:
|
||||
"""
|
||||
Map HuggingFace parameter name to GGUF tensor name.
|
||||
|
||||
This function handles the mismatch between HF parameter naming
|
||||
conventions and gguf-py's expected format:
|
||||
1. Strips 'model.' prefix (common in multimodal models)
|
||||
2. Converts '_weight' suffix to '.weight' (Gemma3 compatibility)
|
||||
3. Searches vision_name_map for multimodal parameters
|
||||
4. Falls back to text_name_map for language model parameters
|
||||
|
||||
Args:
|
||||
hf_name: Full HuggingFace parameter name (e.g.,
|
||||
'model.multi_modal_projector.mm_soft_emb_norm.weight')
|
||||
|
||||
Returns:
|
||||
GGUF tensor name with suffix (e.g., 'mm.soft_emb_norm.weight')
|
||||
or None if no mapping found
|
||||
"""
|
||||
# In transformers v5, multimodal models (e.g. Gemma3) wrap
|
||||
# all sub-models under an outer 'model.' attribute, producing
|
||||
# state_dict keys like 'model.language_model.layers.0...' and
|
||||
# 'model.vision_tower.vision_model...'. Strip this outer
|
||||
# prefix so the keys match what gguf-py expects.
|
||||
if is_multimodal and hf_name.startswith("model."):
|
||||
hf_name = hf_name[6:] # Remove outer 'model.'
|
||||
|
||||
# Strip 'language_model.' prefix for multimodal models - gguf-py
|
||||
# tensor mappings expect parameter names without this prefix.
|
||||
# Note: 'model.' prefix should be KEPT for text-only models as
|
||||
# gguf-py expects it.
|
||||
if hf_name.startswith("language_model."):
|
||||
hf_name = hf_name[15:] # Remove 'language_model.'
|
||||
# Re-add 'model.' prefix because gguf-py text tensor maps
|
||||
# expect 'model.layers...' format.
|
||||
if is_multimodal:
|
||||
hf_name = "model." + hf_name
|
||||
|
||||
# Parse parameter name and suffix
|
||||
if hf_name.endswith((".weight", ".bias")):
|
||||
base_name, suffix = hf_name.rsplit(".", 1)
|
||||
else:
|
||||
base_name, suffix = hf_name, ""
|
||||
# Handle '_weight' suffix (Gemma3 naming: parameter ends with
|
||||
# '_weight' instead of '.weight')
|
||||
if base_name.endswith("_weight"):
|
||||
base_name = base_name[:-7] # Remove '_weight'
|
||||
suffix = "weight"
|
||||
|
||||
gguf_name = None
|
||||
# Priority 1: Search vision/projector parameters for multimodal models
|
||||
if vision_name_map is not None:
|
||||
gguf_name = vision_name_map.get_name(base_name)
|
||||
|
||||
# Priority 2: Search text backbone parameters
|
||||
if gguf_name is None:
|
||||
gguf_name = text_name_map.get_name(base_name)
|
||||
|
||||
if gguf_name is None:
|
||||
return None
|
||||
|
||||
return gguf_name + "." + suffix
|
||||
|
||||
# Build mapping and track unmapped parameters
|
||||
unmapped_params = []
|
||||
for hf_name in state_dict:
|
||||
gguf_name_with_suffix = find_hf_name_in_tensor_map(hf_name)
|
||||
|
||||
# Track mapping success
|
||||
if gguf_name_with_suffix is not None:
|
||||
gguf_to_hf_name_map[gguf_name_with_suffix] = hf_name
|
||||
logger.debug("Mapped GGUF %s → HF %s", gguf_name_with_suffix, hf_name)
|
||||
elif hf_name not in gguf_to_hf_name_map.values():
|
||||
# Parameter not in manual overrides either
|
||||
unmapped_params.append(hf_name)
|
||||
|
||||
# All parameters (except those initialized by other means) must be mapped:
|
||||
# both vision/projector and backbone
|
||||
if unmapped_params:
|
||||
unmapped_params = list(
|
||||
filter(
|
||||
lambda x: not any(re.fullmatch(p, x) for p in sideload_params),
|
||||
unmapped_params,
|
||||
)
|
||||
)
|
||||
if unmapped_params:
|
||||
raise RuntimeError(
|
||||
f"Failed to map GGUF parameters "
|
||||
f"({len(unmapped_params)}): "
|
||||
f"{unmapped_params}"
|
||||
)
|
||||
return gguf_to_hf_name_map
|
||||
|
||||
def _get_gguf_weight_type(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
model_name_or_path: str,
|
||||
gguf_to_hf_name_map: dict[str, str],
|
||||
) -> dict[str, str]:
|
||||
gguf_files = self._get_all_gguf_files(model_name_or_path)
|
||||
weight_type_map = {}
|
||||
for f in gguf_files:
|
||||
weight_type_map.update(get_gguf_weight_type_map(f, gguf_to_hf_name_map))
|
||||
is_multimodal = hasattr(model_config.hf_config, "vision_config")
|
||||
if is_multimodal:
|
||||
mmproj_file = detect_gguf_multimodal(model_name_or_path)
|
||||
assert mmproj_file is not None, (
|
||||
"Could not find mm_proj file for multimodal GGUF model"
|
||||
)
|
||||
logger.info("Loading extra mm_proj weights from %s...", mmproj_file)
|
||||
mm_proj_weight_type_map = get_gguf_weight_type_map(
|
||||
mmproj_file, gguf_to_hf_name_map
|
||||
)
|
||||
weight_type_map.update(mm_proj_weight_type_map)
|
||||
return weight_type_map
|
||||
|
||||
def _get_weights_iterator(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
model_name_or_path: str,
|
||||
gguf_to_hf_name_map: dict[str, str],
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""
|
||||
Iterate over GGUF model weights, loading from both main model file and
|
||||
mmproj.gguf for multimodal Gemma3 models.
|
||||
|
||||
For Gemma3 multimodal GGUF models:
|
||||
- Main file (gemma-3-*.gguf): Language model weights (model.*)
|
||||
- mmproj file (mmproj*.gguf): Vision tower + projector weights (v.*, mm.*)
|
||||
|
||||
Yields:
|
||||
Tuples of (parameter_name, tensor) for all model weights
|
||||
"""
|
||||
hf_config = model_config.hf_config
|
||||
is_multimodal = hasattr(hf_config, "vision_config")
|
||||
|
||||
if is_multimodal:
|
||||
# Load mm_proj (mm_encoder + projector) for multimodal weights
|
||||
mmproj_file = detect_gguf_multimodal(model_name_or_path)
|
||||
assert mmproj_file is not None, (
|
||||
"Could not find mm_proj file for multimodal GGUF model"
|
||||
)
|
||||
yield from gguf_quant_weights_iterator(mmproj_file, gguf_to_hf_name_map)
|
||||
|
||||
gguf_files = self._get_all_gguf_files(model_name_or_path)
|
||||
if len(gguf_files) > 1:
|
||||
yield from gguf_quant_weights_iterator_multi(
|
||||
gguf_files, gguf_to_hf_name_map
|
||||
)
|
||||
else:
|
||||
yield from gguf_quant_weights_iterator(
|
||||
model_name_or_path, gguf_to_hf_name_map
|
||||
)
|
||||
|
||||
def download_model(self, model_config: ModelConfig) -> None:
|
||||
self._prepare_weights(model_config)
|
||||
|
||||
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
|
||||
local_model_path = self._prepare_weights(model_config)
|
||||
gguf_weights_map = self._get_gguf_weights_map(model_config)
|
||||
model.load_weights(
|
||||
self._get_weights_iterator(model_config, local_model_path, gguf_weights_map)
|
||||
)
|
||||
|
||||
def load_model(
|
||||
self, vllm_config: VllmConfig, model_config: ModelConfig, prefix: str = ""
|
||||
) -> nn.Module:
|
||||
device_config = vllm_config.device_config
|
||||
local_model_path = self._prepare_weights(model_config)
|
||||
gguf_weights_map = self._get_gguf_weights_map(model_config)
|
||||
# we can only know if tie word embeddings after mapping weights
|
||||
gguf_files = self._get_all_gguf_files(local_model_path)
|
||||
all_extra_names = []
|
||||
for f in gguf_files:
|
||||
all_extra_names.extend(get_gguf_extra_tensor_names(f, gguf_weights_map))
|
||||
if "lm_head.weight" in all_extra_names:
|
||||
model_config.hf_config.update({"tie_word_embeddings": True})
|
||||
|
||||
weight_type_map = self._get_gguf_weight_type(
|
||||
model_config, local_model_path, gguf_weights_map
|
||||
)
|
||||
# filter out unquantized modules to skip
|
||||
unquant_names = [
|
||||
name.removesuffix(".weight")
|
||||
for name, weight_type in weight_type_map.items()
|
||||
if weight_type in ("F32", "F16", "BF16") and name.endswith(".weight")
|
||||
]
|
||||
logger.debug("GGUF unquantized modules: %s", unquant_names)
|
||||
if TYPE_CHECKING:
|
||||
vllm_config.quant_config = cast(GGUFConfig, vllm_config.quant_config)
|
||||
vllm_config.quant_config.unquantized_modules.extend(unquant_names)
|
||||
|
||||
target_device = torch.device(device_config.device)
|
||||
with set_default_torch_dtype(model_config.dtype):
|
||||
with target_device:
|
||||
model = initialize_model(vllm_config=vllm_config, prefix=prefix)
|
||||
self.load_weights(model, model_config)
|
||||
|
||||
process_weights_after_loading(model, model_config, target_device)
|
||||
return model
|
||||
@@ -54,11 +54,6 @@ except ImportError:
|
||||
runai_model_streamer = PlaceholderModule("runai_model_streamer") # type: ignore[assignment]
|
||||
SafetensorsStreamer = runai_model_streamer.placeholder_attr("SafetensorsStreamer")
|
||||
|
||||
try:
|
||||
import gguf
|
||||
except ImportError:
|
||||
gguf = PlaceholderModule("gguf")
|
||||
|
||||
try:
|
||||
from fastsafetensors import SafeTensorsFileLoader, SingleGroup
|
||||
except ImportError:
|
||||
@@ -250,10 +245,6 @@ def get_quant_config(
|
||||
raise ValueError("Model quantization method is not specified in the config.")
|
||||
quant_cls = get_quantization_config(model_config.quantization)
|
||||
|
||||
# GGUF doesn't have config file
|
||||
if model_config.quantization == "gguf":
|
||||
return quant_cls()
|
||||
|
||||
# Read the quantization config from the HF model config, if available.
|
||||
hf_quant_config = getattr(model_config.hf_config, "quantization_config", None)
|
||||
# some vision model may keep quantization_config in their text_config
|
||||
@@ -437,52 +428,6 @@ def get_sparse_attention_config(
|
||||
return config
|
||||
|
||||
|
||||
def download_gguf(
|
||||
repo_id: str,
|
||||
quant_type: str,
|
||||
cache_dir: str | None = None,
|
||||
revision: str | None = None,
|
||||
ignore_patterns: str | list[str] | None = None,
|
||||
) -> str:
|
||||
# Use patterns that snapshot_download can handle directly
|
||||
# Patterns to match:
|
||||
# - *-{quant_type}.gguf (root)
|
||||
# - *-{quant_type}-*.gguf (root sharded)
|
||||
# - */*-{quant_type}.gguf (subdir)
|
||||
# - */*-{quant_type}-*.gguf (subdir sharded)
|
||||
allow_patterns = [
|
||||
f"*-{quant_type}.gguf",
|
||||
f"*-{quant_type}-*.gguf",
|
||||
f"*/*-{quant_type}.gguf",
|
||||
f"*/*-{quant_type}-*.gguf",
|
||||
]
|
||||
|
||||
# Use download_weights_from_hf which handles caching and downloading
|
||||
folder = download_weights_from_hf(
|
||||
model_name_or_path=repo_id,
|
||||
cache_dir=cache_dir,
|
||||
allow_patterns=allow_patterns,
|
||||
revision=revision,
|
||||
ignore_patterns=ignore_patterns,
|
||||
)
|
||||
|
||||
# Find the downloaded file(s) in the folder
|
||||
local_files = []
|
||||
for pattern in allow_patterns:
|
||||
# Convert pattern to glob pattern for local filesystem
|
||||
glob_pattern = os.path.join(folder, pattern)
|
||||
local_files.extend(glob.glob(glob_pattern))
|
||||
|
||||
if not local_files:
|
||||
raise ValueError(
|
||||
f"Downloaded GGUF files not found in {folder} for quant_type {quant_type}"
|
||||
)
|
||||
|
||||
# Sort to ensure consistent ordering (prefer non-sharded files)
|
||||
local_files.sort(key=lambda x: (x.count("-"), x))
|
||||
return local_files[0]
|
||||
|
||||
|
||||
@instrument(span_name="Download weights - HF")
|
||||
def download_weights_from_hf(
|
||||
model_name_or_path: str,
|
||||
@@ -1237,118 +1182,6 @@ def multi_thread_pt_weights_iterator(
|
||||
del state
|
||||
|
||||
|
||||
def get_gguf_extra_tensor_names(
|
||||
gguf_file: str | Path, gguf_to_hf_name_map: dict[str, str]
|
||||
) -> list[str]:
|
||||
reader = gguf.GGUFReader(gguf_file)
|
||||
expected_gguf_keys = set(gguf_to_hf_name_map.keys())
|
||||
exact_gguf_keys = set([tensor.name for tensor in reader.tensors])
|
||||
extra_keys = expected_gguf_keys - exact_gguf_keys
|
||||
return [gguf_to_hf_name_map[key] for key in extra_keys]
|
||||
|
||||
|
||||
def get_gguf_weight_type_map(
|
||||
gguf_file: str | Path, gguf_to_hf_name_map: dict[str, str]
|
||||
) -> dict[str, str]:
|
||||
"""
|
||||
Return GGUF mapped weight's name and its quant type
|
||||
"""
|
||||
reader = gguf.GGUFReader(gguf_file)
|
||||
return {
|
||||
gguf_to_hf_name_map[tensor.name]: tensor.tensor_type.name
|
||||
for tensor in reader.tensors
|
||||
if tensor.name in gguf_to_hf_name_map
|
||||
}
|
||||
|
||||
|
||||
def gguf_quant_weights_iterator(
|
||||
gguf_file: str | Path, gguf_to_hf_name_map: dict[str, str]
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""
|
||||
Iterate over the quant weights in the model gguf files and convert
|
||||
them to torch tensors.
|
||||
Be careful of the order of yielding weight types and weights data,
|
||||
we have to yield all weight types first before yielding any weights.
|
||||
Otherwise it would cause issue when loading weights with for packed
|
||||
layer with different quant types.
|
||||
"""
|
||||
|
||||
reader = gguf.GGUFReader(gguf_file)
|
||||
|
||||
for tensor in reader.tensors:
|
||||
if tensor.name in gguf_to_hf_name_map:
|
||||
weight_type = tensor.tensor_type
|
||||
name = gguf_to_hf_name_map[tensor.name]
|
||||
|
||||
if weight_type.name not in ("F32", "BF16", "F16"):
|
||||
weight_type_name = name.replace("weight", "qweight_type")
|
||||
weight_type = torch.tensor(weight_type)
|
||||
yield weight_type_name, weight_type
|
||||
|
||||
for tensor in reader.tensors:
|
||||
if tensor.name in gguf_to_hf_name_map:
|
||||
weight = tensor.data
|
||||
weight_type = tensor.tensor_type
|
||||
name = gguf_to_hf_name_map[tensor.name]
|
||||
if weight_type.name not in ("F32", "BF16", "F16"):
|
||||
name = name.replace("weight", "qweight")
|
||||
if weight_type.name == "BF16" and tensor.data.dtype == np.uint8:
|
||||
# BF16 is currently the only "quantization" type that isn't
|
||||
# actually quantized but is read as a raw byte tensor.
|
||||
# Reinterpret as `torch.bfloat16` tensor.
|
||||
weight = weight.view(np.uint16)
|
||||
if reader.byte_order == "S":
|
||||
# GGUF endianness != system endianness
|
||||
weight = weight.byteswap()
|
||||
param = torch.tensor(weight).view(torch.bfloat16)
|
||||
else:
|
||||
param = torch.tensor(weight)
|
||||
yield name, param
|
||||
|
||||
|
||||
def gguf_quant_weights_iterator_multi(
|
||||
gguf_files: list[str], gguf_to_hf_name_map: dict[str, str]
|
||||
) -> Generator[tuple[str, torch.Tensor], None, None]:
|
||||
"""
|
||||
Iterate over the quant weights across multiple GGUF shard files
|
||||
and convert them to torch tensors.
|
||||
|
||||
Like gguf_quant_weights_iterator, we yield all weight types first
|
||||
before yielding any weights data to avoid issues with packed layers
|
||||
that have different quant types.
|
||||
"""
|
||||
readers = [gguf.GGUFReader(f) for f in gguf_files]
|
||||
|
||||
# First pass: yield all weight types across all shards
|
||||
for reader in readers:
|
||||
for tensor in reader.tensors:
|
||||
if tensor.name in gguf_to_hf_name_map:
|
||||
weight_type = tensor.tensor_type
|
||||
name = gguf_to_hf_name_map[tensor.name]
|
||||
if weight_type.name not in ("F32", "BF16", "F16"):
|
||||
weight_type_name = name.replace("weight", "qweight_type")
|
||||
weight_type = torch.tensor(weight_type)
|
||||
yield weight_type_name, weight_type
|
||||
|
||||
# Second pass: yield all weight data across all shards
|
||||
for reader in readers:
|
||||
for tensor in reader.tensors:
|
||||
if tensor.name in gguf_to_hf_name_map:
|
||||
weight = tensor.data
|
||||
weight_type = tensor.tensor_type
|
||||
name = gguf_to_hf_name_map[tensor.name]
|
||||
if weight_type.name not in ("F32", "BF16", "F16"):
|
||||
name = name.replace("weight", "qweight")
|
||||
if weight_type.name == "BF16" and tensor.data.dtype == np.uint8:
|
||||
weight = weight.view(np.uint16)
|
||||
if reader.byte_order == "S":
|
||||
weight = weight.byteswap()
|
||||
param = torch.tensor(weight).view(torch.bfloat16)
|
||||
else:
|
||||
param = torch.tensor(weight)
|
||||
yield name, param
|
||||
|
||||
|
||||
def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
|
||||
"""convert PySafeSlice object from safetensors to torch.Tensor
|
||||
|
||||
|
||||
@@ -228,9 +228,6 @@ class ApertusAttention(nn.Module):
|
||||
quant_config: QuantizationConfig | None,
|
||||
) -> None:
|
||||
is_neox_style = True
|
||||
is_gguf = quant_config and quant_config.get_name() == "gguf"
|
||||
if is_gguf and config.model_type == "apertus":
|
||||
is_neox_style = False
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
|
||||
@@ -162,8 +162,6 @@ class ExaoneAttention(nn.Module):
|
||||
)
|
||||
|
||||
is_neox_style = True
|
||||
if quant_config is not None and quant_config.get_name() == "gguf":
|
||||
is_neox_style = False
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
|
||||
@@ -168,8 +168,6 @@ class Exaone4Attention(nn.Module):
|
||||
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
is_neox_style = True
|
||||
if quant_config is not None and quant_config.get_name() == "gguf":
|
||||
is_neox_style = False
|
||||
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
is_sliding = config.layer_types[layer_idx] == "sliding_attention"
|
||||
|
||||
@@ -377,15 +377,6 @@ class Gemma3Model(nn.Module):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
# Revert +1 during llama.cpp conversion
|
||||
# see: https://github.com/ggml-org/llama.cpp/blob/be7c3034108473beda214fd1d7c98fd6a7a3bdf5/convert_hf_to_gguf.py#L3397-L3400
|
||||
if (
|
||||
self.quant_config
|
||||
and self.quant_config.get_name() == "gguf"
|
||||
and name.endswith("norm.weight")
|
||||
):
|
||||
loaded_weight -= 1
|
||||
|
||||
# Check if this is a scale parameter that needs remapping first
|
||||
if name.endswith((".k_scale", ".v_scale", ".q_scale", ".prob_scale")):
|
||||
# Try to remap the scale name first
|
||||
|
||||
@@ -161,9 +161,6 @@ class Jais2Attention(nn.Module):
|
||||
)
|
||||
|
||||
is_neox_style = True
|
||||
if quant_config is not None and quant_config.get_name() == "gguf":
|
||||
is_neox_style = False
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
|
||||
@@ -239,9 +239,6 @@ class LlamaAttention(nn.Module):
|
||||
quant_config: QuantizationConfig | None,
|
||||
) -> None:
|
||||
is_neox_style = True
|
||||
is_gguf = quant_config and quant_config.get_name() == "gguf"
|
||||
if is_gguf and config.model_type == "llama":
|
||||
is_neox_style = False
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
|
||||
@@ -238,9 +238,6 @@ class Llama4Attention(nn.Module):
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
is_neox_style = True
|
||||
is_gguf = quant_config and quant_config.get_name() == "gguf"
|
||||
if is_gguf and config.model_type == "llama":
|
||||
is_neox_style = False
|
||||
|
||||
self.rotary_emb = (
|
||||
get_rope(
|
||||
|
||||
@@ -279,12 +279,14 @@ class OlmoeModel(nn.Module):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
self.config = config
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
|
||||
@@ -517,10 +517,6 @@ class OpenPanguEmbeddedAttention(nn.Module):
|
||||
quant_config: QuantizationConfig | None,
|
||||
) -> None:
|
||||
is_neox_style = True
|
||||
is_gguf = quant_config and quant_config.get_name() == "gguf"
|
||||
if is_gguf and config.model_type == "PanguEmbedded":
|
||||
is_neox_style = False
|
||||
|
||||
rope_parameters = config.rope_parameters or {}
|
||||
if rope_parameters is not None and rope_parameters.get(
|
||||
"mrope_interleaved", False
|
||||
@@ -716,20 +712,6 @@ class OpenPanguSinkAttention(nn.Module):
|
||||
# no need to narrow
|
||||
is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
|
||||
|
||||
# Special case for GGUF
|
||||
is_gguf_weight = getattr(param, "is_gguf_weight", False)
|
||||
is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
|
||||
if is_gguf_weight_type:
|
||||
param.weight_type = loaded_weight.item()
|
||||
|
||||
# Materialize GGUF UninitializedParameter
|
||||
if is_gguf_weight and isinstance(param, nn.UninitializedParameter):
|
||||
final_shape = list(loaded_weight.shape)
|
||||
if output_dim is not None:
|
||||
assert final_shape[output_dim] % self.tp_size == 0
|
||||
final_shape[output_dim] = final_shape[output_dim] // self.tp_size
|
||||
param.materialize(final_shape, dtype=loaded_weight.dtype)
|
||||
|
||||
param_data = param.data
|
||||
if output_dim is not None and not is_sharded_weight:
|
||||
shard_size = param_data.shape[output_dim]
|
||||
|
||||
@@ -952,38 +952,12 @@ class SiglipVisionModel(nn.Module):
|
||||
break
|
||||
else:
|
||||
param = params_dict[name]
|
||||
param = maybe_swap_ffn_param(
|
||||
name, param, loaded_weight, params_dict, self.quant_config
|
||||
)
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
def maybe_swap_ffn_param(
|
||||
name: str,
|
||||
param: torch.Tensor,
|
||||
loaded_weight: torch.Tensor,
|
||||
params_dict: dict[str, torch.Tensor],
|
||||
quant_config: QuantizationConfig,
|
||||
) -> torch.Tensor:
|
||||
if not (quant_config and quant_config.get_name() == "gguf") or ".fc" not in name:
|
||||
return param
|
||||
# Some GGUF models have fc1 and fc2 weights swapped
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
output_dim = getattr(param, "output_dim", 0)
|
||||
output_size = param.size(output_dim) * tp_size
|
||||
weight_out_size = loaded_weight.size(output_dim)
|
||||
if ".fc1." in name and output_size != weight_out_size:
|
||||
new_name = name.replace(".fc1.", ".fc2.")
|
||||
param = params_dict[new_name]
|
||||
elif ".fc2." in name and output_size != weight_out_size:
|
||||
new_name = name.replace(".fc2.", ".fc1.")
|
||||
param = params_dict[new_name]
|
||||
return param
|
||||
|
||||
|
||||
# Adapted from: https://github.com/huggingface/transformers/blob/v4.54.1/src/transformers/models/siglip/modeling_siglip.py#L200
|
||||
class SiglipTextEmbeddings(nn.Module):
|
||||
def __init__(self, config: SiglipTextConfig):
|
||||
|
||||
@@ -448,7 +448,6 @@ class RocmPlatform(Platform):
|
||||
"deepseek_v4_fp8",
|
||||
"compressed-tensors",
|
||||
"fbgemm_fp8",
|
||||
"gguf",
|
||||
"quark",
|
||||
"mxfp4",
|
||||
"mxfp8",
|
||||
|
||||
@@ -12,13 +12,6 @@ from typing_extensions import TypeVar, assert_never
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.config import get_config
|
||||
from vllm.transformers_utils.gguf_utils import (
|
||||
check_gguf_file,
|
||||
get_gguf_file_path_from_hf,
|
||||
is_gguf,
|
||||
is_remote_gguf,
|
||||
split_remote_gguf,
|
||||
)
|
||||
from vllm.transformers_utils.repo_utils import (
|
||||
any_pattern_in_repo_files,
|
||||
is_mistral_model_repo,
|
||||
@@ -124,21 +117,6 @@ def resolve_tokenizer_args(
|
||||
)
|
||||
tokenizer_name = tokenizer_path
|
||||
|
||||
# Separate model folder from file path for GGUF models
|
||||
if is_gguf(tokenizer_name):
|
||||
if check_gguf_file(tokenizer_name):
|
||||
kwargs["gguf_file"] = Path(tokenizer_name).name
|
||||
tokenizer_name = Path(tokenizer_name).parent
|
||||
elif is_remote_gguf(tokenizer_name):
|
||||
tokenizer_name, quant_type = split_remote_gguf(tokenizer_name)
|
||||
# Get the HuggingFace Hub path for the GGUF file
|
||||
gguf_file = get_gguf_file_path_from_hf(
|
||||
tokenizer_name,
|
||||
quant_type,
|
||||
revision=revision,
|
||||
)
|
||||
kwargs["gguf_file"] = gguf_file
|
||||
|
||||
if "truncation_side" not in kwargs:
|
||||
if runner_type == "generate" or runner_type == "draft":
|
||||
kwargs["truncation_side"] = "left"
|
||||
|
||||
@@ -19,7 +19,6 @@ from transformers import GenerationConfig, PretrainedConfig
|
||||
from transformers.configuration_utils import ALLOWED_LAYER_TYPES
|
||||
from transformers.models.auto.image_processing_auto import get_image_processor_config
|
||||
from transformers.models.auto.modeling_auto import (
|
||||
MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
|
||||
MODEL_MAPPING_NAMES,
|
||||
)
|
||||
from transformers.models.auto.tokenization_auto import get_tokenizer_config
|
||||
@@ -35,12 +34,6 @@ from vllm.transformers_utils.utils import (
|
||||
from vllm.utils.torch_utils import common_broadcastable_dtype
|
||||
|
||||
from .config_parser_base import ConfigParserBase
|
||||
from .gguf_utils import (
|
||||
check_gguf_file,
|
||||
is_gguf,
|
||||
is_remote_gguf,
|
||||
split_remote_gguf,
|
||||
)
|
||||
from .repo_utils import (
|
||||
file_or_path_exists,
|
||||
get_hf_file_to_dict,
|
||||
@@ -611,17 +604,9 @@ def maybe_override_with_speculators(
|
||||
Returns:
|
||||
Tuple of (resolved_model, resolved_tokenizer, speculative_config)
|
||||
"""
|
||||
if check_gguf_file(model):
|
||||
kwargs["gguf_file"] = Path(model).name
|
||||
gguf_model_repo = Path(model).parent
|
||||
elif is_remote_gguf(model):
|
||||
repo_id, _ = split_remote_gguf(model)
|
||||
gguf_model_repo = Path(repo_id)
|
||||
else:
|
||||
gguf_model_repo = None
|
||||
kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
|
||||
config_dict, _ = PretrainedConfig.get_config_dict(
|
||||
model if gguf_model_repo is None else gguf_model_repo,
|
||||
model,
|
||||
revision=revision,
|
||||
token=hf_token,
|
||||
**without_trust_remote_code(kwargs),
|
||||
@@ -659,21 +644,6 @@ def get_config(
|
||||
hf_overrides_fn: Callable[[PretrainedConfig], PretrainedConfig] | None = None,
|
||||
**kwargs,
|
||||
) -> PretrainedConfig:
|
||||
# Separate model folder from file path for GGUF models
|
||||
|
||||
_is_gguf = is_gguf(model)
|
||||
_is_remote_gguf = is_remote_gguf(model)
|
||||
if _is_gguf:
|
||||
if check_gguf_file(model):
|
||||
# Local GGUF file
|
||||
kwargs["gguf_file"] = Path(model).name
|
||||
model = Path(model).parent
|
||||
elif _is_remote_gguf:
|
||||
# Remote GGUF - extract repo_id from repo_id:quant_type format
|
||||
# The actual GGUF file will be downloaded later by GGUFModelLoader
|
||||
# Keep model as repo_id:quant_type for download, but use repo_id for config
|
||||
model, _ = split_remote_gguf(model)
|
||||
|
||||
if config_format == "auto":
|
||||
try:
|
||||
# First check for Mistral to avoid defaulting to
|
||||
@@ -684,25 +654,8 @@ def get_config(
|
||||
model=model, config_name=MISTRAL_CONFIG_NAME, revision=revision
|
||||
):
|
||||
config_format = "mistral"
|
||||
elif (_is_gguf and not _is_remote_gguf) or file_or_path_exists(
|
||||
model, HF_CONFIG_NAME, revision=revision
|
||||
):
|
||||
elif file_or_path_exists(model, HF_CONFIG_NAME, revision=revision):
|
||||
config_format = "hf"
|
||||
# Remote GGUF models must have config.json in repo,
|
||||
# otherwise the config can't be parsed correctly.
|
||||
# FIXME(Isotr0py): Support remote GGUF repos without config.json
|
||||
elif _is_remote_gguf and not file_or_path_exists(
|
||||
model, HF_CONFIG_NAME, revision=revision
|
||||
):
|
||||
err_msg = (
|
||||
"Could not find config.json for remote GGUF model repo. "
|
||||
"To load remote GGUF model through `<repo_id>:<quant_type>`, "
|
||||
"ensure your model has config.json (HF format) file. "
|
||||
"Otherwise please specify --hf-config-path <original_repo> "
|
||||
"in engine args to fetch config from unquantized hf model."
|
||||
)
|
||||
logger.error(err_msg)
|
||||
raise ValueError(err_msg)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Could not detect config format for no config file found. "
|
||||
@@ -737,34 +690,6 @@ def get_config(
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# Patching defaults for GGUF models
|
||||
if _is_gguf:
|
||||
# Some models have different default values between GGUF and HF.
|
||||
def apply_gguf_default(key: str, gguf_default: Any):
|
||||
"""
|
||||
Apply GGUF defaults unless explicitly configured.
|
||||
|
||||
This function reads/writes external `config` and `config_dict`.
|
||||
If the specified `key` is not in `config_dict` (i.e. not explicitly
|
||||
configured and the default HF value is used), it updates the
|
||||
corresponding `config` value to `gguf_default`.
|
||||
"""
|
||||
if key not in config_dict:
|
||||
config.update({key: gguf_default})
|
||||
|
||||
# Apply architecture-specific GGUF defaults.
|
||||
if config.model_type in {"qwen3_moe"}:
|
||||
# Qwen3 MoE: norm_topk_prob is always true.
|
||||
# Note that, this parameter is always false (HF default) on Qwen2 MoE.
|
||||
apply_gguf_default("norm_topk_prob", True)
|
||||
|
||||
# Special architecture mapping check for GGUF models
|
||||
if _is_gguf:
|
||||
if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
|
||||
raise RuntimeError(f"Can't get gguf config for {config.model_type}.")
|
||||
model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
|
||||
config.update({"architectures": [model_type]})
|
||||
|
||||
# Architecture mapping for models without explicit architectures field
|
||||
if not config.architectures:
|
||||
if config.model_type not in MODEL_MAPPING_NAMES:
|
||||
@@ -856,9 +781,6 @@ def get_pooling_config(
|
||||
A dictionary containing the pooling type and whether
|
||||
normalization is used, or None if no pooling configuration is found.
|
||||
"""
|
||||
if is_remote_gguf(model):
|
||||
model, _ = split_remote_gguf(model)
|
||||
|
||||
modules_file_name = "modules.json"
|
||||
|
||||
modules_dict = None
|
||||
@@ -1074,11 +996,6 @@ def get_hf_image_processor_config(
|
||||
# ModelScope does not provide an interface for image_processor
|
||||
if envs.VLLM_USE_MODELSCOPE:
|
||||
return dict()
|
||||
# Separate model folder from file path for GGUF models
|
||||
if check_gguf_file(model):
|
||||
model = Path(model).parent
|
||||
elif is_remote_gguf(model):
|
||||
model, _ = split_remote_gguf(model)
|
||||
return get_image_processor_config(
|
||||
model, token=hf_token, revision=revision, **kwargs
|
||||
)
|
||||
@@ -1108,13 +1025,6 @@ def try_get_generation_config(
|
||||
config_format: str | ConfigFormat = "auto",
|
||||
hf_token: bool | str | None = None,
|
||||
) -> GenerationConfig | None:
|
||||
# GGUF files don't have generation_config.json - their config is embedded
|
||||
# in the file header. Skip all filesystem lookups to avoid re-reading the
|
||||
# memory-mapped file, which can hang in multi-process scenarios when the
|
||||
# EngineCore process already has the file mapped.
|
||||
if is_gguf(model):
|
||||
return None
|
||||
|
||||
try:
|
||||
return GenerationConfig.from_pretrained(
|
||||
model,
|
||||
|
||||
@@ -1,336 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GGUF utility functions."""
|
||||
|
||||
from functools import cache
|
||||
from os import PathLike
|
||||
from pathlib import Path
|
||||
|
||||
import gguf
|
||||
import regex as re
|
||||
from gguf.constants import Keys, VisionProjectorType
|
||||
from gguf.quants import GGMLQuantizationType
|
||||
from transformers import Gemma3Config, PretrainedConfig, SiglipVisionConfig
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .repo_utils import list_filtered_repo_files
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@cache
|
||||
def check_gguf_file(model: str | PathLike) -> bool:
|
||||
"""Check if the file is a GGUF model."""
|
||||
model = Path(model)
|
||||
if not model.is_file():
|
||||
return False
|
||||
elif model.suffix == ".gguf":
|
||||
return True
|
||||
|
||||
try:
|
||||
with model.open("rb") as f:
|
||||
header = f.read(4)
|
||||
|
||||
return header == b"GGUF"
|
||||
except Exception as e:
|
||||
logger.debug("Error reading file %s: %s", model, e)
|
||||
return False
|
||||
|
||||
|
||||
@cache
|
||||
def is_remote_gguf(model: str | Path) -> bool:
|
||||
"""Check if the model is a remote GGUF model.
|
||||
|
||||
Recognizes two forms:
|
||||
1. Standard: ``repo_id:quant_type`` where *quant_type* is a known
|
||||
GGML quantization type (e.g. ``Q4_K_M``).
|
||||
2. Non-standard: ``repo_id:quant_type`` where *quant_type* contains
|
||||
a known GGML type with extra prefixes (e.g. ``UD-Q4_K_XL``).
|
||||
A warning is logged and actual file existence is validated later
|
||||
during download.
|
||||
"""
|
||||
pattern = r"^[a-zA-Z0-9][a-zA-Z0-9._-]*/[a-zA-Z0-9][a-zA-Z0-9._-]*:[A-Za-z0-9_+-]+$"
|
||||
model = str(model)
|
||||
if re.fullmatch(pattern, model):
|
||||
_, quant_type = model.rsplit(":", 1)
|
||||
if is_valid_gguf_quant_type(quant_type):
|
||||
return True
|
||||
if is_nonstandard_gguf_quant_type(quant_type):
|
||||
logger.warning(
|
||||
"Non-standard GGUF quant type '%s' detected.",
|
||||
quant_type,
|
||||
)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_nonstandard_gguf_quant_type(quant_type: str) -> bool:
|
||||
"""Check if a non-standard quant type contains a known GGML type.
|
||||
|
||||
Splits the quant type by the last ``-`` and checks whether the
|
||||
trailing part is a standard GGML type. For example::
|
||||
|
||||
UD-Q4_K_XL → rsplit → ["UD", "Q4_K_XL"] → Q4_K_XL valid ✓
|
||||
UD-IQ4_NL → rsplit → ["UD", "IQ4_NL"] → IQ4_NL valid ✓
|
||||
Custom-UD-Q4_K → rsplit → ["Custom-UD", "Q4_K"] → Q4_K valid ✓
|
||||
RANDOM → no "-" → False
|
||||
"""
|
||||
if "-" not in quant_type:
|
||||
return False
|
||||
_, remainder = quant_type.rsplit("-", 1)
|
||||
return is_valid_gguf_quant_type(remainder)
|
||||
|
||||
|
||||
# Common suffixes used in GGUF file naming conventions
|
||||
# e.g., Q4_K_M, Q3_K_S, Q5_K_L, Q2_K_XL
|
||||
_GGUF_QUANT_SUFFIXES = ("_M", "_S", "_L", "_XL", "_XS", "_XXS")
|
||||
|
||||
|
||||
def is_valid_gguf_quant_type(gguf_quant_type: str) -> bool:
|
||||
"""Check if the quant type is a valid GGUF quant type.
|
||||
|
||||
Supports both exact GGML quant types (e.g., Q4_K, IQ1_S) and
|
||||
extended naming conventions (e.g., Q4_K_M, Q3_K_S, Q5_K_L).
|
||||
"""
|
||||
# Check for exact match first
|
||||
if getattr(GGMLQuantizationType, gguf_quant_type, None) is not None:
|
||||
return True
|
||||
|
||||
# Check for extended naming conventions (e.g., Q4_K_M -> Q4_K)
|
||||
for suffix in _GGUF_QUANT_SUFFIXES:
|
||||
if gguf_quant_type.endswith(suffix):
|
||||
base_type = gguf_quant_type[: -len(suffix)]
|
||||
if getattr(GGMLQuantizationType, base_type, None) is not None:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def split_remote_gguf(model: str | Path) -> tuple[str, str]:
|
||||
"""Split the model into repo_id and quant type."""
|
||||
model = str(model)
|
||||
if is_remote_gguf(model):
|
||||
parts = model.rsplit(":", 1)
|
||||
return (parts[0], parts[1])
|
||||
raise ValueError(
|
||||
f"Wrong GGUF model or invalid GGUF quant type: {model}.\n"
|
||||
"- It should be in repo_id:quant_type format.\n"
|
||||
f"- Valid base quant types: {GGMLQuantizationType._member_names_}\n"
|
||||
f"- Extended suffixes also supported: {_GGUF_QUANT_SUFFIXES}\n"
|
||||
"- Non-standard GGUF quant types also supported: "
|
||||
"dash-separated prefixes (e.g. UD-Q4_K_XL, Custom-Q8_0)",
|
||||
)
|
||||
|
||||
|
||||
def is_gguf(model: str | Path) -> bool:
|
||||
"""Check if the model is a GGUF model.
|
||||
|
||||
Args:
|
||||
model: Model name, path, or Path object to check.
|
||||
|
||||
Returns:
|
||||
True if the model is a GGUF model, False otherwise.
|
||||
"""
|
||||
model = str(model)
|
||||
|
||||
# Check if it's a local GGUF file
|
||||
if check_gguf_file(model):
|
||||
return True
|
||||
|
||||
# Check if it's a remote GGUF model (repo_id:quant_type format)
|
||||
return is_remote_gguf(model)
|
||||
|
||||
|
||||
def detect_gguf_multimodal(model: str) -> Path | None:
|
||||
"""Check if GGUF model has multimodal projector file.
|
||||
|
||||
Args:
|
||||
model: Model path string
|
||||
|
||||
Returns:
|
||||
Path to mmproj file if found, None otherwise
|
||||
"""
|
||||
if not model.endswith(".gguf"):
|
||||
return None
|
||||
|
||||
try:
|
||||
model_path = Path(model)
|
||||
if not model_path.is_file():
|
||||
return None
|
||||
|
||||
model_dir = model_path.parent
|
||||
mmproj_patterns = ["mmproj.gguf", "mmproj-*.gguf", "*mmproj*.gguf"]
|
||||
for pattern in mmproj_patterns:
|
||||
mmproj_files = list(model_dir.glob(pattern))
|
||||
if mmproj_files:
|
||||
return mmproj_files[0]
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def extract_vision_config_from_gguf(mmproj_path: str) -> "SiglipVisionConfig | None":
|
||||
"""Extract vision config parameters from mmproj.gguf metadata.
|
||||
|
||||
Reads vision encoder configuration from GGUF metadata fields using
|
||||
standardized GGUF constants. Automatically detects the projector type
|
||||
(e.g., gemma3, llama4) and applies model-specific parameters accordingly.
|
||||
|
||||
The function extracts standard CLIP vision parameters from GGUF metadata
|
||||
and applies projector-type-specific customizations. For unknown projector
|
||||
types, it uses safe defaults from SiglipVisionConfig.
|
||||
|
||||
Args:
|
||||
mmproj_path: Path to mmproj.gguf file (str or Path)
|
||||
|
||||
Returns:
|
||||
SiglipVisionConfig if extraction succeeds, None if any required
|
||||
field is missing from the GGUF metadata
|
||||
|
||||
Raises:
|
||||
Exception: Exceptions from GGUF reading (file not found, corrupted
|
||||
file, etc.) propagate directly from gguf.GGUFReader
|
||||
"""
|
||||
reader = gguf.GGUFReader(str(mmproj_path))
|
||||
|
||||
# Detect projector type to apply model-specific parameters
|
||||
projector_type = None
|
||||
projector_type_field = reader.get_field(Keys.Clip.PROJECTOR_TYPE)
|
||||
if projector_type_field:
|
||||
try:
|
||||
projector_type = bytes(projector_type_field.parts[-1]).decode("utf-8")
|
||||
except (AttributeError, UnicodeDecodeError) as e:
|
||||
logger.warning("Failed to decode projector type from GGUF: %s", e)
|
||||
|
||||
# Map GGUF field constants to SiglipVisionConfig parameters.
|
||||
# Uses official GGUF constants from gguf-py for standardization.
|
||||
# Format: {gguf_constant: (param_name, dtype)}
|
||||
VISION_CONFIG_FIELDS = {
|
||||
Keys.ClipVision.EMBEDDING_LENGTH: ("hidden_size", int),
|
||||
Keys.ClipVision.FEED_FORWARD_LENGTH: ("intermediate_size", int),
|
||||
Keys.ClipVision.BLOCK_COUNT: ("num_hidden_layers", int),
|
||||
Keys.ClipVision.Attention.HEAD_COUNT: ("num_attention_heads", int),
|
||||
Keys.ClipVision.IMAGE_SIZE: ("image_size", int),
|
||||
Keys.ClipVision.PATCH_SIZE: ("patch_size", int),
|
||||
Keys.ClipVision.Attention.LAYERNORM_EPS: ("layer_norm_eps", float),
|
||||
}
|
||||
|
||||
# Extract and validate all required fields
|
||||
config_params = {}
|
||||
for gguf_key, (param_name, dtype) in VISION_CONFIG_FIELDS.items():
|
||||
field = reader.get_field(gguf_key)
|
||||
if field is None:
|
||||
logger.warning(
|
||||
"Missing required vision config field '%s' in mmproj.gguf",
|
||||
gguf_key,
|
||||
)
|
||||
return None
|
||||
# Extract scalar value from GGUF field and convert to target type
|
||||
config_params[param_name] = dtype(field.parts[-1])
|
||||
|
||||
# Apply model-specific parameters based on projector type
|
||||
if projector_type == VisionProjectorType.GEMMA3:
|
||||
# Gemma3 doesn't use the vision pooling head (multihead attention)
|
||||
# This is a vLLM-specific parameter used in SiglipVisionTransformer
|
||||
config_params["vision_use_head"] = False
|
||||
logger.info("Detected Gemma3 projector, disabling vision pooling head")
|
||||
# Add other projector-type-specific customizations here as needed
|
||||
# elif projector_type == VisionProjectorType.LLAMA4:
|
||||
# config_params["vision_use_head"] = ...
|
||||
|
||||
# Create config with extracted parameters
|
||||
# Note: num_channels and attention_dropout use SiglipVisionConfig defaults
|
||||
# (3 and 0.0 respectively) which are correct for all models
|
||||
config = SiglipVisionConfig(**config_params)
|
||||
|
||||
if projector_type:
|
||||
logger.info(
|
||||
"Extracted vision config from mmproj.gguf (projector_type: %s)",
|
||||
projector_type,
|
||||
)
|
||||
else:
|
||||
logger.info("Extracted vision config from mmproj.gguf metadata")
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def maybe_patch_hf_config_from_gguf(
|
||||
model: str,
|
||||
hf_config: PretrainedConfig,
|
||||
) -> PretrainedConfig:
|
||||
"""Patch HF config for GGUF models.
|
||||
|
||||
Applies GGUF-specific patches to HuggingFace config:
|
||||
1. For multimodal models: patches architecture and vision config
|
||||
2. For all GGUF models: overrides vocab_size from embedding tensor
|
||||
|
||||
This ensures compatibility with GGUF models that have extended
|
||||
vocabularies (e.g., Unsloth) where the GGUF file contains more
|
||||
tokens than the HuggingFace tokenizer config specifies.
|
||||
|
||||
Args:
|
||||
model: Model path string
|
||||
hf_config: HuggingFace config to patch in-place
|
||||
|
||||
Returns:
|
||||
Updated HuggingFace config
|
||||
"""
|
||||
# Patch multimodal config if mmproj.gguf exists
|
||||
mmproj_path = detect_gguf_multimodal(model)
|
||||
if mmproj_path is not None:
|
||||
vision_config = extract_vision_config_from_gguf(str(mmproj_path))
|
||||
|
||||
# Create HF config for Gemma3 multimodal
|
||||
text_config = hf_config.get_text_config()
|
||||
is_gemma3 = hf_config.model_type in ("gemma3", "gemma3_text")
|
||||
if vision_config is not None and is_gemma3:
|
||||
new_hf_config = Gemma3Config(
|
||||
text_config=text_config,
|
||||
vision_config=vision_config,
|
||||
architectures=["Gemma3ForConditionalGeneration"],
|
||||
)
|
||||
hf_config = new_hf_config
|
||||
|
||||
return hf_config
|
||||
|
||||
|
||||
def get_gguf_file_path_from_hf(
|
||||
repo_id: str | Path,
|
||||
quant_type: str,
|
||||
revision: str | None = None,
|
||||
) -> str:
|
||||
"""Get the GGUF file path from HuggingFace Hub based on repo_id and quant_type.
|
||||
|
||||
Args:
|
||||
repo_id: The HuggingFace repository ID (e.g., "Qwen/Qwen3-0.6B")
|
||||
quant_type: The quantization type (e.g., "Q4_K_M", "F16")
|
||||
revision: Optional revision/branch name
|
||||
|
||||
Returns:
|
||||
The path to the GGUF file on HuggingFace Hub (e.g., "filename.gguf"),
|
||||
"""
|
||||
repo_id = str(repo_id)
|
||||
gguf_patterns = [
|
||||
f"*-{quant_type}.gguf",
|
||||
f"*-{quant_type}-*.gguf",
|
||||
f"*/*-{quant_type}.gguf",
|
||||
f"*/*-{quant_type}-*.gguf",
|
||||
]
|
||||
matching_files = list_filtered_repo_files(
|
||||
repo_id,
|
||||
allow_patterns=gguf_patterns,
|
||||
revision=revision,
|
||||
)
|
||||
|
||||
if len(matching_files) == 0:
|
||||
raise ValueError(
|
||||
"Could not find GGUF file for repo %s with quantization %s.",
|
||||
repo_id,
|
||||
quant_type,
|
||||
)
|
||||
|
||||
# Sort to ensure consistent ordering (prefer non-sharded files)
|
||||
matching_files.sort(key=lambda x: (x.count("-"), x))
|
||||
gguf_filename = matching_files[0]
|
||||
return gguf_filename
|
||||
@@ -25,7 +25,6 @@ from typing_extensions import TypeVar
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils import processors
|
||||
from vllm.transformers_utils.gguf_utils import is_gguf
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
|
||||
from vllm.transformers_utils.utils import convert_model_repo_to_path
|
||||
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
|
||||
@@ -181,17 +180,8 @@ _cached_get_video_processor_cls_name = lru_cache(
|
||||
def get_video_processor_cls_name(
|
||||
model_config: "ModelConfig",
|
||||
) -> str | None:
|
||||
if is_gguf(model_config.model):
|
||||
assert not is_gguf(model_config.tokenizer), (
|
||||
"For multimodal GGUF models, the original tokenizer "
|
||||
"should be used to correctly load video processor metadata."
|
||||
)
|
||||
model = model_config.tokenizer
|
||||
revision = model_config.tokenizer_revision
|
||||
else:
|
||||
model = model_config.model
|
||||
revision = model_config.revision
|
||||
|
||||
model = model_config.model
|
||||
revision = model_config.revision
|
||||
return _cached_get_video_processor_cls_name(model, revision=revision)
|
||||
|
||||
|
||||
@@ -375,20 +365,9 @@ def cached_processor_from_config(
|
||||
processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin,
|
||||
**kwargs: Any,
|
||||
) -> _P:
|
||||
if is_gguf(model_config.model):
|
||||
assert not is_gguf(model_config.tokenizer), (
|
||||
"For multimodal GGUF models, the original tokenizer "
|
||||
"should be used to correctly load processor."
|
||||
)
|
||||
model = model_config.tokenizer
|
||||
revision = model_config.tokenizer_revision
|
||||
else:
|
||||
model = model_config.model
|
||||
revision = model_config.revision
|
||||
|
||||
return cached_get_processor_without_dynamic_kwargs(
|
||||
model,
|
||||
revision=revision,
|
||||
model_config.model,
|
||||
revision=model_config.revision,
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
processor_cls=processor_cls, # type: ignore[arg-type]
|
||||
**_merge_mm_kwargs(model_config, processor_cls, **kwargs),
|
||||
@@ -489,19 +468,9 @@ def cached_image_processor_from_config(
|
||||
model_config: "ModelConfig",
|
||||
**kwargs: Any,
|
||||
):
|
||||
if is_gguf(model_config.model):
|
||||
assert not is_gguf(model_config.tokenizer), (
|
||||
"For multimodal GGUF models, the original tokenizer "
|
||||
"should be used to correctly load image processor."
|
||||
)
|
||||
model = model_config.tokenizer
|
||||
revision = model_config.tokenizer_revision
|
||||
else:
|
||||
model = model_config.model
|
||||
revision = model_config.revision
|
||||
return cached_get_image_processor(
|
||||
model,
|
||||
revision=revision,
|
||||
model_config.model,
|
||||
revision=model_config.revision,
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
**_merge_mm_kwargs(model_config, AutoImageProcessor, **kwargs),
|
||||
)
|
||||
|
||||
@@ -66,7 +66,6 @@ _QUANT_WEIGHT_BYTE_SIZE: dict[str, float] = {
|
||||
"bitsandbytes": 0.5,
|
||||
"modelopt_fp4": 0.5,
|
||||
"petit_nvfp4": 0.5,
|
||||
"gguf": 0.5,
|
||||
"compressed-tensors": 0.5,
|
||||
"torchao": 0.5,
|
||||
"quark": 0.5,
|
||||
|
||||
Reference in New Issue
Block a user