[Misc] Consolidate csrc shared headers and prune dead code (#60721)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -5,7 +5,7 @@
|
||||
#include <hip/hip_bf16.h>
|
||||
#include <hip/hip_bfloat16.h>
|
||||
|
||||
#include "../../../../attention/attention_dtypes.h"
|
||||
#include "core/vec_dtypes/vec_dtypes.h"
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
|
||||
namespace vllm {
|
||||
|
||||
@@ -0,0 +1,506 @@
|
||||
#include "common.cuh"
|
||||
#include "core/cuda_compat.h"
|
||||
#include "../../../dispatch_utils.h"
|
||||
#include "../../../cub_helpers.h"
|
||||
#include "../../vectorization_utils.cuh"
|
||||
#include "../../../torch_utils.h"
|
||||
#include <torch/csrc/stable/macros.h>
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
namespace vllm {
|
||||
|
||||
namespace {
|
||||
|
||||
// 16 fp8 outputs form one 16-byte store; the matching 16-element input load
|
||||
// is 32 B for 16-bit types and 64 B for fp32.
|
||||
constexpr int kFp8QuantVecSize = 16;
|
||||
constexpr int kFp8QuantBlockSize = 256;
|
||||
|
||||
template <typename T>
|
||||
using fp8_quant_vec_t = vec_n_t<T, kFp8QuantVecSize>;
|
||||
|
||||
template <typename T>
|
||||
bool ptr_vec_aligned(const T* ptr) {
|
||||
return reinterpret_cast<uintptr_t>(ptr) % alignof(fp8_quant_vec_t<T>) == 0;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float warp_reduce_max(float v) {
|
||||
#pragma unroll
|
||||
for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) {
|
||||
v = fmaxf(v, VLLM_SHFL_XOR_SYNC(v, offset));
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
// Block-wide max, returned to every thread. Warp partials go through one
|
||||
// shared-memory exchange, so the whole reduction costs a single
|
||||
// __syncthreads. blockDim.x must be a multiple of WARP_SIZE and at most
|
||||
// kFp8QuantBlockSize.
|
||||
__device__ __forceinline__ float block_reduce_max(float v, float* warp_max) {
|
||||
const int lane = threadIdx.x % WARP_SIZE;
|
||||
const int warp = threadIdx.x / WARP_SIZE;
|
||||
const int num_warps = blockDim.x / WARP_SIZE;
|
||||
v = warp_reduce_max(v);
|
||||
if (lane == 0) {
|
||||
warp_max[warp] = v;
|
||||
}
|
||||
__syncthreads();
|
||||
v = warp_max[0];
|
||||
for (int w = 1; w < num_warps; ++w) {
|
||||
v = fmaxf(v, warp_max[w]);
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// STRIDE_I_ZERO: true if scale_stride_i == 0 (per-tensor or per-channel)
|
||||
// STRIDE_J_ZERO: true if scale_stride_j == 0 (per-tensor or per-token)
|
||||
template <typename scalar_t, typename fp8_type, bool STRIDE_I_ZERO,
|
||||
bool STRIDE_J_ZERO>
|
||||
__global__ void scaled_fp8_quant_kernel_strided_group_shape(
|
||||
fp8_type* __restrict__ out, const scalar_t* __restrict__ input,
|
||||
const float* __restrict__ scale, int hidden_size, int64_t in_row_stride,
|
||||
int64_t out_row_stride, int group_m, int group_n, int64_t scale_stride_i,
|
||||
int64_t scale_stride_j) {
|
||||
const int64_t token_idx = blockIdx.x;
|
||||
const int tid = threadIdx.x;
|
||||
|
||||
const scalar_t* token_in = input + token_idx * in_row_stride;
|
||||
fp8_type* token_out = out + token_idx * out_row_stride;
|
||||
|
||||
// Precompute row-level base offset for scale access (compile-time eliminated
|
||||
// when STRIDE_I_ZERO)
|
||||
const int64_t scale_row_base =
|
||||
STRIDE_I_ZERO ? 0
|
||||
: static_cast<int>(token_idx) / group_m * scale_stride_i;
|
||||
|
||||
auto get_inv_scale = [&](int gj) {
|
||||
return 1.0f / scale[scale_row_base + gj * scale_stride_j];
|
||||
};
|
||||
|
||||
int cached_gj = -1;
|
||||
float cached_inv_scale = 0.0f;
|
||||
auto get_inv_scale_cached = [&](int gj) {
|
||||
if (gj != cached_gj) {
|
||||
cached_inv_scale = 1.0f / scale[scale_row_base + gj * scale_stride_j];
|
||||
cached_gj = gj;
|
||||
}
|
||||
return cached_inv_scale;
|
||||
};
|
||||
|
||||
constexpr int VEC_SIZE = 16; // FP8 so vectorize to 128 bits
|
||||
auto scaled_fp8_conversion_vectorized = [&](const scalar_t* in, fp8_type* out,
|
||||
int size, float inv_scale) {
|
||||
vectorize_with_alignment<VEC_SIZE>(
|
||||
in, out, size, tid, blockDim.x,
|
||||
[=] __device__(fp8_type & dst, const scalar_t& src) {
|
||||
dst = scaled_fp8_conversion<true, fp8_type>(static_cast<float>(src),
|
||||
inv_scale);
|
||||
});
|
||||
};
|
||||
|
||||
if (STRIDE_J_ZERO && hidden_size % VEC_SIZE == 0) {
|
||||
// Per-tensor or per-token: single scale per row, vectorize full row
|
||||
scaled_fp8_conversion_vectorized(token_in, token_out, hidden_size,
|
||||
get_inv_scale(0));
|
||||
} else if (group_n % VEC_SIZE == 0) {
|
||||
// Multiple column groups with vectorization
|
||||
const int num_groups_n = hidden_size / group_n;
|
||||
|
||||
for (int gj = 0; gj < num_groups_n; gj++) {
|
||||
scaled_fp8_conversion_vectorized(token_in + gj * group_n,
|
||||
token_out + gj * group_n, group_n,
|
||||
get_inv_scale(gj));
|
||||
}
|
||||
} else {
|
||||
// Scalar path for small column groups (group_n < VEC_SIZE)
|
||||
for (int n = tid; n < hidden_size; n += blockDim.x) {
|
||||
const int gj = n / group_n;
|
||||
token_out[n] = scaled_fp8_conversion<true, fp8_type>(
|
||||
static_cast<float>(token_in[n]), get_inv_scale_cached(gj));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Per-tensor abs-max of a contiguous tensor viewed as a flat array of
|
||||
// 16-element vectors: grid-stride loop, one atomic per block.
|
||||
template <typename scalar_t, typename fp8_type>
|
||||
__global__ void __launch_bounds__(kFp8QuantBlockSize)
|
||||
absmax_reduction_flat(float* __restrict__ scale,
|
||||
const scalar_t* __restrict__ input,
|
||||
int64_t num_vecs) {
|
||||
using in_vec_t = fp8_quant_vec_t<scalar_t>;
|
||||
const auto* v_in = reinterpret_cast<const in_vec_t*>(input);
|
||||
const int64_t stride = static_cast<int64_t>(gridDim.x) * blockDim.x;
|
||||
|
||||
float thread_max = 0.0f;
|
||||
#pragma unroll 4
|
||||
for (int64_t v = static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x;
|
||||
v < num_vecs; v += stride) {
|
||||
const in_vec_t vec = v_in[v];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kFp8QuantVecSize; ++i) {
|
||||
thread_max = fmaxf(thread_max, fabsf(static_cast<float>(vec.val[i])));
|
||||
}
|
||||
}
|
||||
|
||||
__shared__ float warp_max[kFp8QuantBlockSize / 32];
|
||||
const float block_max = block_reduce_max(thread_max, warp_max);
|
||||
if (threadIdx.x == 0) {
|
||||
atomicMaxFloat(scale, block_max / quant_type_max_v<fp8_type>);
|
||||
}
|
||||
}
|
||||
|
||||
// Per-tensor abs-max for strided or unaligned rows: one block per token.
|
||||
template <typename scalar_t, typename fp8_type>
|
||||
__global__ void __launch_bounds__(kFp8QuantBlockSize)
|
||||
segmented_max_reduction_strided(float* __restrict__ scale,
|
||||
const scalar_t* __restrict__ input,
|
||||
int hidden_size, int64_t in_row_stride,
|
||||
int64_t num_tokens) {
|
||||
const int64_t token_idx = blockIdx.x;
|
||||
if (token_idx >= num_tokens) {
|
||||
return;
|
||||
}
|
||||
const scalar_t* row_ptr = input + token_idx * in_row_stride;
|
||||
|
||||
float thread_max = 0.0f;
|
||||
vectorize_read_with_alignment<kFp8QuantVecSize>(
|
||||
row_ptr, hidden_size, threadIdx.x, blockDim.x,
|
||||
[&] __device__(const scalar_t& v) {
|
||||
thread_max = fmaxf(thread_max, fabsf(static_cast<float>(v)));
|
||||
});
|
||||
|
||||
__shared__ float warp_max[kFp8QuantBlockSize / 32];
|
||||
const float block_max = block_reduce_max(thread_max, warp_max);
|
||||
if (threadIdx.x == 0) {
|
||||
atomicMaxFloat(scale, block_max / quant_type_max_v<fp8_type>);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t, typename fp8_type>
|
||||
__global__ void scaled_fp8_quant_kernel_strided_dynamic(
|
||||
fp8_type* __restrict__ out, const scalar_t* __restrict__ input,
|
||||
const float* __restrict__ scale, int hidden_size, int64_t in_row_stride,
|
||||
int64_t out_row_stride) {
|
||||
const int64_t token_idx = blockIdx.x;
|
||||
const int tid = threadIdx.x;
|
||||
|
||||
const scalar_t* token_in = input + token_idx * in_row_stride;
|
||||
fp8_type* token_out = out + token_idx * out_row_stride;
|
||||
|
||||
const float reciprocal_scale = 1.0f / (*scale);
|
||||
vectorize_with_alignment<16>(
|
||||
token_in, token_out, hidden_size, tid, blockDim.x,
|
||||
[=] __device__(fp8_type & dst, const scalar_t& src) {
|
||||
dst = scaled_fp8_conversion<true, fp8_type>(static_cast<float>(src),
|
||||
reciprocal_scale);
|
||||
});
|
||||
}
|
||||
|
||||
template <typename scalar_t, typename fp8_type>
|
||||
__global__ void dynamic_per_token_scaled_fp8_quant_kernel_strided(
|
||||
fp8_type* __restrict__ out, float* __restrict__ scale,
|
||||
const scalar_t* __restrict__ input, const float* __restrict__ scale_ub,
|
||||
int hidden_size, int64_t in_row_stride, int64_t out_row_stride) {
|
||||
const int64_t token_idx = blockIdx.x;
|
||||
const int tid = threadIdx.x;
|
||||
|
||||
// Use int64 to avoid overflowing an int32 when calculating this offset
|
||||
int64_t in_offset = static_cast<int64_t>(token_idx) * in_row_stride;
|
||||
int64_t out_offset = static_cast<int64_t>(token_idx) * out_row_stride;
|
||||
const scalar_t* token_in = input + in_offset;
|
||||
fp8_type* token_out = out + out_offset;
|
||||
|
||||
// 1) per-token absmax
|
||||
float absmax_val = 0.f;
|
||||
vectorize_read_with_alignment<16>(
|
||||
token_in, hidden_size, tid, blockDim.x, [&] __device__(scalar_t v) {
|
||||
absmax_val = fmaxf(absmax_val, fabsf(static_cast<float>(v)));
|
||||
});
|
||||
|
||||
using BlockReduce = cub::BlockReduce<float, 256>;
|
||||
__shared__ typename BlockReduce::TempStorage tmp;
|
||||
const float block_max =
|
||||
BlockReduce(tmp).Reduce(absmax_val, CubMaxOp{}, blockDim.x);
|
||||
|
||||
__shared__ float token_scale;
|
||||
if (tid == 0) {
|
||||
token_scale = scale_ub ? fminf(block_max, *scale_ub) : block_max;
|
||||
token_scale = fmaxf(token_scale / quant_type_max_v<fp8_type>,
|
||||
min_scaling_factor<fp8_type>::val());
|
||||
scale[token_idx] = token_scale;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// 2) quantize
|
||||
vectorize_with_alignment<16>(
|
||||
token_in, token_out, hidden_size, tid, blockDim.x,
|
||||
[=] __device__(fp8_type & dst, const scalar_t& src) {
|
||||
dst = scaled_fp8_conversion<false, fp8_type>(static_cast<float>(src),
|
||||
token_scale);
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace vllm
|
||||
|
||||
void static_scaled_fp8_quant(
|
||||
torch::stable::Tensor& out, // [..., d]
|
||||
torch::stable::Tensor const& input, // [..., d]
|
||||
torch::stable::Tensor const& scale, // various shapes
|
||||
std::optional<torch::headeronly::IntHeaderOnlyArrayRef>
|
||||
opt_group_shape) // optional explicit [group_m, group_n]
|
||||
{
|
||||
STD_TORCH_CHECK(input.stride(-1) == 1,
|
||||
"last dimension of input must be contiguous");
|
||||
STD_TORCH_CHECK(out.stride(-1) == 1,
|
||||
"last dimension of output must be contiguous");
|
||||
|
||||
const int hidden_size = input.size(-1); // N (columns)
|
||||
const int num_tokens = input.numel() / hidden_size; // M (rows)
|
||||
|
||||
// Determine group_m, group_n, and scale strides from scale shape
|
||||
// Scale indexing: scale[gi * scale_stride_j + gj * scale_stride_i]
|
||||
// where gi = m / group_m, gj = n / group_n
|
||||
int group_m, group_n;
|
||||
int64_t scale_stride_i, scale_stride_j;
|
||||
|
||||
if (scale.dim() == 0 || scale.numel() == 1) {
|
||||
// Per-tensor: one scale for the entire tensor
|
||||
group_m = num_tokens;
|
||||
group_n = hidden_size;
|
||||
scale_stride_i = 0;
|
||||
scale_stride_j = 0;
|
||||
} else if (scale.dim() == 1) {
|
||||
// 1D scale: require explicit group_shape to disambiguate per-channel vs
|
||||
// per-token (avoids edge case where num_tokens == hidden_size)
|
||||
STD_TORCH_CHECK(
|
||||
opt_group_shape.has_value(),
|
||||
"1D scale requires explicit group_shape to disambiguate "
|
||||
"per-channel vs per-token quantization. "
|
||||
"Use group_shape=(-1, 1) for per-channel or group_shape=(1, "
|
||||
"-1) for per-token.");
|
||||
STD_TORCH_CHECK(opt_group_shape->size() == 2,
|
||||
"group_shape must have exactly 2 elements, got ",
|
||||
opt_group_shape->size());
|
||||
|
||||
const auto opt_group_m = (*opt_group_shape)[0];
|
||||
const auto opt_group_n = (*opt_group_shape)[1];
|
||||
group_m = opt_group_m == -1 ? num_tokens : static_cast<int>(opt_group_m);
|
||||
group_n = opt_group_n == -1 ? hidden_size : static_cast<int>(opt_group_n);
|
||||
|
||||
// Validate the explicit group shape matches the 1D scale
|
||||
const int64_t scale_len = scale.numel();
|
||||
const int64_t expected_scale_m = num_tokens / group_m;
|
||||
const int64_t expected_scale_n = hidden_size / group_n;
|
||||
const int64_t expected_scale_numel = expected_scale_m * expected_scale_n;
|
||||
|
||||
STD_TORCH_CHECK(scale_len == expected_scale_numel, "1D scale length (",
|
||||
scale_len, ") does not match expected size (",
|
||||
expected_scale_numel, ") for group_shape (", opt_group_m,
|
||||
", ", opt_group_n, ") with input shape (", num_tokens, ", ",
|
||||
hidden_size, ")");
|
||||
|
||||
// For 1D scale, determine strides based on which dim is trivial
|
||||
// Scale indexing: scale[gi * scale_stride_i + gj * scale_stride_j]
|
||||
// where gi = m / group_m (row group), gj = n / group_n (col group)
|
||||
if (expected_scale_m == 1) {
|
||||
// Per-channel style: one scale in M dim, scale varies along N
|
||||
// gi = 0 always, gj varies, so stride_1 traverses the scale
|
||||
scale_stride_i = 0;
|
||||
scale_stride_j = scale.stride(0);
|
||||
} else if (expected_scale_n == 1) {
|
||||
// Per-token style: one scale in N dim, scale varies along M
|
||||
// gj = 0 always, gi varies, so stride_0 traverses the scale
|
||||
scale_stride_i = scale.stride(0);
|
||||
scale_stride_j = 0;
|
||||
} else {
|
||||
STD_TORCH_CHECK(
|
||||
false,
|
||||
"1D scale can only be used when one of the scale dimensions is 1. "
|
||||
"For 2D group scaling, use a 2D scale tensor.");
|
||||
}
|
||||
} else if (scale.dim() == 2) {
|
||||
// 2D scale: infer group sizes from scale dimensions (or use explicit if
|
||||
// provided)
|
||||
const int64_t scale_size_0 = scale.size(0);
|
||||
const int64_t scale_size_1 = scale.size(1);
|
||||
|
||||
STD_TORCH_CHECK(num_tokens % scale_size_0 == 0, "num_tokens (", num_tokens,
|
||||
") must be divisible by scale.size(0) (", scale_size_0,
|
||||
")");
|
||||
STD_TORCH_CHECK(hidden_size % scale_size_1 == 0, "hidden_size (",
|
||||
hidden_size, ") must be divisible by scale.size(1) (",
|
||||
scale_size_1, ")");
|
||||
|
||||
// Infer from 2D scale shape
|
||||
int inferred_group_m = num_tokens / scale_size_0;
|
||||
int inferred_group_n = hidden_size / scale_size_1;
|
||||
|
||||
// Use explicit if provided, otherwise use inferred
|
||||
if (opt_group_shape.has_value()) {
|
||||
STD_TORCH_CHECK(opt_group_shape->size() == 2,
|
||||
"group_shape must have exactly 2 elements, got ",
|
||||
opt_group_shape->size());
|
||||
const auto opt_group_m = (*opt_group_shape)[0];
|
||||
const auto opt_group_n = (*opt_group_shape)[1];
|
||||
group_m = opt_group_m == -1 ? num_tokens : static_cast<int>(opt_group_m);
|
||||
group_n = opt_group_n == -1 ? hidden_size : static_cast<int>(opt_group_n);
|
||||
|
||||
// Validate explicit matches inferred
|
||||
STD_TORCH_CHECK(
|
||||
group_m == inferred_group_m && group_n == inferred_group_n,
|
||||
"Explicit group_shape (", opt_group_m, ", ", opt_group_n,
|
||||
") does not match inferred group shape (", inferred_group_m, ", ",
|
||||
inferred_group_n, ") from 2D scale tensor shape (", scale_size_0,
|
||||
", ", scale_size_1, ")");
|
||||
} else {
|
||||
group_m = inferred_group_m;
|
||||
group_n = inferred_group_n;
|
||||
}
|
||||
|
||||
scale_stride_i = scale.stride(0);
|
||||
scale_stride_j = scale.stride(1);
|
||||
} else {
|
||||
STD_TORCH_CHECK(false, "scale must be 0D, 1D, or 2D tensor, but got ",
|
||||
scale.dim(), "D");
|
||||
}
|
||||
|
||||
const int block_size = 256;
|
||||
dim3 grid(num_tokens);
|
||||
dim3 block(block_size);
|
||||
|
||||
const int64_t in_row_stride = input.stride(-2);
|
||||
const int64_t out_row_stride = out.stride(-2);
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
|
||||
// Dispatch to template-specialized kernel based on stride pattern
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "scaled_fp8_quant_kernel_scalar_type", [&] {
|
||||
VLLM_STABLE_DISPATCH_FP8_TYPES(
|
||||
out.scalar_type(), "scaled_fp8_quant_kernel_fp8_type", [&] {
|
||||
VLLM_STABLE_DISPATCH_BOOL(scale_stride_i == 0, S0_ZERO, [&] {
|
||||
VLLM_STABLE_DISPATCH_BOOL(scale_stride_j == 0, S1_ZERO, [&] {
|
||||
vllm::scaled_fp8_quant_kernel_strided_group_shape<
|
||||
scalar_t, fp8_t, S0_ZERO, S1_ZERO>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<fp8_t>(),
|
||||
input.const_data_ptr<scalar_t>(),
|
||||
scale.const_data_ptr<float>(), hidden_size,
|
||||
in_row_stride, out_row_stride, group_m, group_n,
|
||||
scale_stride_i, scale_stride_j);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void dynamic_scaled_fp8_quant(torch::stable::Tensor& out, // [..., d]
|
||||
torch::stable::Tensor const& input, // [..., d]
|
||||
torch::stable::Tensor& scale) // [1]
|
||||
{
|
||||
STD_TORCH_CHECK(input.stride(-1) == 1,
|
||||
"last dimension of input must be contiguous");
|
||||
STD_TORCH_CHECK(out.stride(-1) == 1,
|
||||
"last dimension of output must be contiguous");
|
||||
|
||||
const int hidden_size = input.size(-1);
|
||||
const int num_tokens = input.numel() / hidden_size;
|
||||
const int block_size = vllm::kFp8QuantBlockSize;
|
||||
dim3 grid(num_tokens);
|
||||
dim3 block(block_size);
|
||||
|
||||
const int64_t in_row_stride = input.stride(-2);
|
||||
const int64_t out_row_stride = out.stride(-2);
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
|
||||
// scale tensor should be initialised to <=0 before reduction
|
||||
STD_CUDA_CHECK(cudaMemsetAsync(scale.mutable_data_ptr<float>(), 0,
|
||||
sizeof(float), stream));
|
||||
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "scaled_fp8_quant_kernel_scalar_type", [&] {
|
||||
VLLM_STABLE_DISPATCH_FP8_TYPES(
|
||||
out.scalar_type(), "scaled_fp8_quant_kernel_fp8_type", [&] {
|
||||
const scalar_t* in_ptr = input.const_data_ptr<scalar_t>();
|
||||
const bool flat = in_row_stride == hidden_size &&
|
||||
hidden_size % vllm::kFp8QuantVecSize == 0 &&
|
||||
vllm::ptr_vec_aligned(in_ptr);
|
||||
if (flat) {
|
||||
const int64_t num_vecs = input.numel() / vllm::kFp8QuantVecSize;
|
||||
const int64_t max_blocks =
|
||||
static_cast<int64_t>(
|
||||
get_device_prop()->multiProcessorCount) *
|
||||
8;
|
||||
const int64_t num_blocks = std::max<int64_t>(
|
||||
1,
|
||||
std::min<int64_t>(
|
||||
max_blocks, (num_vecs + block_size - 1) / block_size));
|
||||
vllm::absmax_reduction_flat<scalar_t, fp8_t>
|
||||
<<<num_blocks, block, 0, stream>>>(
|
||||
scale.mutable_data_ptr<float>(), in_ptr, num_vecs);
|
||||
} else {
|
||||
vllm::segmented_max_reduction_strided<scalar_t, fp8_t>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
scale.mutable_data_ptr<float>(), in_ptr, hidden_size,
|
||||
in_row_stride, static_cast<int64_t>(num_tokens));
|
||||
}
|
||||
|
||||
vllm::scaled_fp8_quant_kernel_strided_dynamic<scalar_t, fp8_t>
|
||||
<<<grid, block, 0, stream>>>(out.mutable_data_ptr<fp8_t>(),
|
||||
input.const_data_ptr<scalar_t>(),
|
||||
scale.const_data_ptr<float>(),
|
||||
hidden_size, in_row_stride,
|
||||
out_row_stride);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void dynamic_per_token_scaled_fp8_quant(
|
||||
torch::stable::Tensor& out, // [..., d]
|
||||
torch::stable::Tensor const& input, // [..., d]
|
||||
torch::stable::Tensor& scales,
|
||||
std::optional<torch::stable::Tensor> const& scale_ub) {
|
||||
STD_TORCH_CHECK(input.stride(-1) == 1,
|
||||
"last dimension of input must be contiguous");
|
||||
STD_TORCH_CHECK(out.stride(-1) == 1,
|
||||
"last dimension of output must be contiguous");
|
||||
|
||||
const int hidden_size = input.size(-1);
|
||||
const int num_tokens = input.numel() / hidden_size;
|
||||
const int block_size = 256;
|
||||
dim3 grid(num_tokens);
|
||||
dim3 block(std::min(hidden_size, block_size));
|
||||
|
||||
const int64_t in_row_stride = input.stride(-2);
|
||||
const int64_t out_row_stride = out.stride(-2);
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(),
|
||||
"dynamic_per_token_scaled_fp8_quant_kernel_scalar_type", [&] {
|
||||
VLLM_STABLE_DISPATCH_FP8_TYPES(
|
||||
out.scalar_type(),
|
||||
"dynamic_per_token_scaled_fp8_quant_kernel_fp8_type", [&] {
|
||||
vllm::dynamic_per_token_scaled_fp8_quant_kernel_strided<scalar_t,
|
||||
fp8_t>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<fp8_t>(),
|
||||
scales.mutable_data_ptr<float>(),
|
||||
input.const_data_ptr<scalar_t>(),
|
||||
scale_ub.has_value() ? scale_ub->const_data_ptr<float>()
|
||||
: nullptr,
|
||||
hidden_size, in_row_stride, out_row_stride);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -1,17 +1,17 @@
|
||||
#pragma once
|
||||
|
||||
#include "libtorch_stable/quantization/vectorization.cuh"
|
||||
#include "../../utils.cuh"
|
||||
#include "quantization/vectorization.cuh"
|
||||
#include "quantization/utils.cuh"
|
||||
|
||||
#include <cmath>
|
||||
|
||||
// This header is shared between _C and _C_stable_libtorch targets.
|
||||
// This header is shared between _rocm_C and _C_stable_libtorch targets.
|
||||
// torch_utils.h provides get_device_prop(). We need to pass USE_CUDA
|
||||
// to the .so to expose some of the shims used by torch_utils.h. For now
|
||||
// this is only done for _C_stable_libtorch and not for _C, so we use the
|
||||
// non stable at::cuda::getCurrentDeviceProperties for _C for now.
|
||||
// this is only done for _C_stable_libtorch and not for _rocm_C, so we use
|
||||
// the non stable at::cuda::getCurrentDeviceProperties for _rocm_C for now.
|
||||
#ifdef TORCH_TARGET_VERSION
|
||||
#include "../../../libtorch_stable/torch_utils.h"
|
||||
#include "torch_utils.h"
|
||||
#else
|
||||
#ifdef USE_ROCM
|
||||
#include <ATen/hip/HIPContext.h>
|
||||
@@ -19,9 +19,9 @@
|
||||
#endif
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#include "nvidia/quant_utils.cuh"
|
||||
#include "quantization/w8a8/fp8/nvidia/quant_utils.cuh"
|
||||
#else
|
||||
#include "amd/quant_utils.cuh"
|
||||
#include "quantization/w8a8/fp8/amd/quant_utils.cuh"
|
||||
#endif
|
||||
|
||||
// Determines the preferred FP8 type for the current platform.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include "../../../../attention/attention_dtypes.h"
|
||||
#include "core/vec_dtypes/vec_dtypes.h"
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
#include <assert.h>
|
||||
#include <float.h>
|
||||
@@ -243,7 +243,7 @@ __inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, __nv_bfloat16>(
|
||||
}
|
||||
|
||||
// torch stable-ABI (headeronly) scalar types delegate to the CUDA-native
|
||||
// conversions, so libtorch_stable kernels dispatched on c10::BFloat16 /
|
||||
// conversions, so stable-ABI kernels dispatched on c10::BFloat16 /
|
||||
// c10::Half quantize correctly without manual casts.
|
||||
template <>
|
||||
__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, c10::BFloat16>(
|
||||
|
||||
@@ -0,0 +1,881 @@
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
#include <torch/csrc/stable/ops.h>
|
||||
#include <torch/headeronly/util/Exception.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
|
||||
#include <cmath>
|
||||
|
||||
#ifdef USE_ROCM
|
||||
#include <hip/hip_fp8.h>
|
||||
#else
|
||||
#include <cuda_fp8.h>
|
||||
#endif
|
||||
|
||||
#include "quantization/vectorization.cuh"
|
||||
#include "quantization/vectorization_utils.cuh"
|
||||
#include "dispatch_utils.h"
|
||||
#include "torch_utils.h"
|
||||
|
||||
__device__ __forceinline__ float GroupReduceMax(float val) {
|
||||
#ifdef USE_ROCM
|
||||
// 16-thread logical groups may pack up to four per 64-lane wavefront; use a
|
||||
// 64-bit mask and explicit width so shuffles stay within each group.
|
||||
const int lane_in_wave = threadIdx.x % warpSize;
|
||||
const unsigned long long mask = 0xFFFFull << ((lane_in_wave / 16) * 16);
|
||||
val = fmaxf(val, __shfl_xor_sync(mask, val, 8, 16));
|
||||
val = fmaxf(val, __shfl_xor_sync(mask, val, 4, 16));
|
||||
val = fmaxf(val, __shfl_xor_sync(mask, val, 2, 16));
|
||||
val = fmaxf(val, __shfl_xor_sync(mask, val, 1, 16));
|
||||
#else
|
||||
unsigned mask = threadIdx.x % 32 >= 16 ? 0xffff0000 : 0x0000ffff;
|
||||
|
||||
val = fmaxf(val, __shfl_xor_sync(mask, val, 8));
|
||||
val = fmaxf(val, __shfl_xor_sync(mask, val, 4));
|
||||
val = fmaxf(val, __shfl_xor_sync(mask, val, 2));
|
||||
val = fmaxf(val, __shfl_xor_sync(mask, val, 1));
|
||||
#endif
|
||||
return val;
|
||||
}
|
||||
|
||||
template <typename T, bool SCALE_UE8M0>
|
||||
__device__ __forceinline__ float ComputeGroupScale(
|
||||
const T* __restrict__ group_input, T* __restrict__ smem_group,
|
||||
const int group_size, const int lane_id, const int threads_per_group,
|
||||
const float eps, const float max_8bit) {
|
||||
float local_absmax = eps;
|
||||
|
||||
constexpr int vec_size = 16 / sizeof(T);
|
||||
|
||||
// copy global -> shared & compute absmax
|
||||
auto scalar_op_cache = [&] __device__(T & dst, const T& src) {
|
||||
float abs_v = fabsf(static_cast<float>(src));
|
||||
local_absmax = fmaxf(local_absmax, abs_v);
|
||||
dst = src;
|
||||
};
|
||||
|
||||
vllm::vectorize_with_alignment<vec_size>(
|
||||
group_input, // in
|
||||
smem_group, // out (shared)
|
||||
group_size, // elements per group
|
||||
lane_id, // thread id
|
||||
threads_per_group, // stride in group
|
||||
scalar_op_cache); // scalar handler
|
||||
|
||||
local_absmax = GroupReduceMax(local_absmax);
|
||||
|
||||
float y_s = local_absmax / max_8bit;
|
||||
if constexpr (SCALE_UE8M0) {
|
||||
y_s = exp2f(ceilf(log2f(fmaxf(fabsf(y_s), 1e-10f))));
|
||||
}
|
||||
|
||||
return y_s;
|
||||
}
|
||||
|
||||
template <typename T, typename DST_DTYPE>
|
||||
__device__ __forceinline__ void QuantizeGroup(
|
||||
const T* __restrict__ smem_group, DST_DTYPE* __restrict__ group_output,
|
||||
const int group_size, const int lane_id, const int threads_per_group,
|
||||
const float y_s, const float min_8bit, const float max_8bit) {
|
||||
constexpr int vec_size = 16 / sizeof(T);
|
||||
|
||||
// quantize shared -> global 8-bit
|
||||
auto scalar_op_quant = [&] __device__(DST_DTYPE & dst, const T& src) {
|
||||
float q = fminf(fmaxf(static_cast<float>(src) / y_s, min_8bit), max_8bit);
|
||||
dst = DST_DTYPE(q);
|
||||
};
|
||||
|
||||
vllm::vectorize_with_alignment<vec_size>(
|
||||
smem_group, // in (shared)
|
||||
group_output, // out (global quant tensor)
|
||||
group_size, // elements
|
||||
lane_id, // tid
|
||||
threads_per_group, // stride
|
||||
scalar_op_quant); // scalar handler
|
||||
}
|
||||
|
||||
template <typename T, typename DST_DTYPE, bool IS_COLUMN_MAJOR = false,
|
||||
bool SCALE_UE8M0 = false, typename scale_packed_t = float,
|
||||
bool CHECK_BOUNDS = false>
|
||||
__global__ void per_token_group_quant_8bit_kernel(
|
||||
const T* __restrict__ input, void* __restrict__ output_q,
|
||||
scale_packed_t* __restrict__ output_s, const int group_size,
|
||||
const int num_groups, const int groups_per_block, const float eps,
|
||||
const float min_8bit, const float max_8bit, const int scale_num_rows = 0,
|
||||
const int scale_stride = 0) {
|
||||
const int threads_per_group = 16;
|
||||
const int64_t local_group_id = threadIdx.x / threads_per_group;
|
||||
const int lane_id = threadIdx.x % threads_per_group;
|
||||
|
||||
const int64_t block_group_id = blockIdx.x * groups_per_block;
|
||||
const int64_t global_group_id = block_group_id + local_group_id;
|
||||
const int64_t block_group_offset = global_group_id * group_size;
|
||||
|
||||
using scale_element_t = float;
|
||||
static_assert(sizeof(scale_packed_t) % sizeof(scale_element_t) == 0);
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
|
||||
// shared memory to cache each group's data to avoid double DRAM reads.
|
||||
extern __shared__ __align__(16) char smem_raw[];
|
||||
T* smem = reinterpret_cast<T*>(smem_raw);
|
||||
T* smem_group = smem + local_group_id * group_size;
|
||||
const bool valid_group = !CHECK_BOUNDS || global_group_id < num_groups;
|
||||
float y_s = 1.0f;
|
||||
if (valid_group) {
|
||||
const T* group_input = input + block_group_offset;
|
||||
scale_element_t* scale_output;
|
||||
|
||||
if constexpr (IS_COLUMN_MAJOR) {
|
||||
const int num_elems_per_pack =
|
||||
static_cast<int>(sizeof(scale_packed_t) / sizeof(scale_element_t));
|
||||
const int scale_num_rows_element = scale_num_rows * num_elems_per_pack;
|
||||
const int row_idx = global_group_id / scale_num_rows_element;
|
||||
const int col_idx_raw = global_group_id % scale_num_rows_element;
|
||||
const int col_idx = col_idx_raw / num_elems_per_pack;
|
||||
const int pack_idx = col_idx_raw % num_elems_per_pack;
|
||||
scale_output = reinterpret_cast<scale_element_t*>(output_s) +
|
||||
(col_idx * scale_stride * num_elems_per_pack +
|
||||
row_idx * num_elems_per_pack + pack_idx);
|
||||
} else {
|
||||
scale_output = output_s + global_group_id;
|
||||
}
|
||||
|
||||
y_s = ComputeGroupScale<T, SCALE_UE8M0>(group_input, smem_group, group_size,
|
||||
lane_id, threads_per_group, eps,
|
||||
max_8bit);
|
||||
|
||||
scale_element_t y_s_quant = y_s;
|
||||
|
||||
if (lane_id == 0) {
|
||||
*scale_output = y_s_quant;
|
||||
}
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
if (valid_group) {
|
||||
DST_DTYPE* group_output =
|
||||
static_cast<DST_DTYPE*>(output_q) + block_group_offset;
|
||||
QuantizeGroup<T, DST_DTYPE>(smem_group, group_output, group_size, lane_id,
|
||||
threads_per_group, y_s, min_8bit, max_8bit);
|
||||
}
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
}
|
||||
|
||||
// Register-resident variant for the unpacked float32 scale layout, mirroring
|
||||
// per_token_group_quant_8bit_packed_register_kernel below, which float32-scale
|
||||
// callers cannot reach because it is hardwired to UE8M0 exponents packed
|
||||
// 4-per-int32 with TMA-aligned strides.
|
||||
//
|
||||
// A group_size-128 group of 16-bit input is 256 B, so it fits in registers and
|
||||
// the smem staging plus the __syncthreads() that the kernel above needs between
|
||||
// its absmax and quantize passes are both avoidable.
|
||||
//
|
||||
// Requires GROUP_SIZE == THREADS_PER_GROUP * VEC_SIZE, i.e. group_size 128 with
|
||||
// 16-bit input; CanUseRegisterPath enforces that.
|
||||
template <typename T, typename DST_DTYPE, bool IS_COLUMN_MAJOR = false,
|
||||
bool SCALE_UE8M0 = false, int GROUP_SIZE = 128>
|
||||
__global__ void per_token_group_quant_8bit_register_kernel(
|
||||
const T* __restrict__ input, void* __restrict__ output_q,
|
||||
float* __restrict__ output_s, const int64_t num_groups, const float eps,
|
||||
const float min_8bit, const float max_8bit, const int scale_num_rows,
|
||||
const int scale_stride) {
|
||||
constexpr int THREADS_PER_GROUP = 8;
|
||||
constexpr int VEC_SIZE = 32 / sizeof(T);
|
||||
static_assert(32 % THREADS_PER_GROUP == 0,
|
||||
"THREADS_PER_GROUP must divide the warp size so the shuffle "
|
||||
"mask stays inside one group");
|
||||
static_assert(sizeof(DST_DTYPE) == 1,
|
||||
"the uint4 store packs one byte per element");
|
||||
|
||||
// CanUseRegisterPath only routes 16-bit inputs here, but
|
||||
// VLLM_STABLE_DISPATCH_FLOATING_TYPES instantiates every floating type, so
|
||||
// gate the body on the shape this schedule can actually serve instead of a
|
||||
// static_assert.
|
||||
if constexpr (GROUP_SIZE != THREADS_PER_GROUP * VEC_SIZE) {
|
||||
return;
|
||||
} else {
|
||||
const int64_t group_id =
|
||||
(static_cast<int64_t>(blockIdx.x) * blockDim.x + threadIdx.x) /
|
||||
THREADS_PER_GROUP;
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
|
||||
if (group_id >= num_groups) {
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
|
||||
const int lane_id = threadIdx.x % THREADS_PER_GROUP;
|
||||
|
||||
// alignas(16) is required for the uint4 reinterpret_cast: the default
|
||||
// alignof for 16-bit T is 2.
|
||||
alignas(16) T regs[VEC_SIZE];
|
||||
{
|
||||
const T* group_input = input + group_id * GROUP_SIZE + lane_id * VEC_SIZE;
|
||||
uint4* dst = reinterpret_cast<uint4*>(®s[0]);
|
||||
const uint4* src = reinterpret_cast<const uint4*>(group_input);
|
||||
dst[0] = src[0];
|
||||
dst[1] = src[1];
|
||||
}
|
||||
|
||||
float local_absmax = eps;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < VEC_SIZE; ++i) {
|
||||
local_absmax = fmaxf(local_absmax, fabsf(static_cast<float>(regs[i])));
|
||||
}
|
||||
|
||||
// Reduce within the 8-lane subgroup that shares this group.
|
||||
#ifdef USE_ROCM
|
||||
const int lane_in_wave = threadIdx.x % warpSize;
|
||||
const unsigned long long mask = 0xFFull << (lane_in_wave & ~7);
|
||||
local_absmax =
|
||||
fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 4, 8));
|
||||
local_absmax =
|
||||
fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 2, 8));
|
||||
local_absmax =
|
||||
fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 1, 8));
|
||||
#else
|
||||
const unsigned mask = 0xffu << (threadIdx.x & 24u);
|
||||
local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 4));
|
||||
local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 2));
|
||||
local_absmax = fmaxf(local_absmax, __shfl_xor_sync(mask, local_absmax, 1));
|
||||
#endif
|
||||
|
||||
float y_s = local_absmax / max_8bit;
|
||||
if constexpr (SCALE_UE8M0) {
|
||||
// Bit-exact with exp2f(ceilf(log2f(fmaxf(fabsf(y_s), 1e-10f)))): bump the
|
||||
// exponent whenever the mantissa is non-zero, then clear the mantissa.
|
||||
y_s = fmaxf(fabsf(y_s), 1e-10f);
|
||||
const uint32_t bits = __float_as_uint(y_s);
|
||||
const uint32_t exp_bits = (bits >> 23) & 0xffu;
|
||||
const uint32_t mant_bits = bits & 0x7fffffu;
|
||||
y_s = __uint_as_float((exp_bits + (mant_bits != 0u ? 1u : 0u)) << 23);
|
||||
}
|
||||
|
||||
if (lane_id == 0) {
|
||||
if constexpr (IS_COLUMN_MAJOR) {
|
||||
const int64_t row_idx = group_id / scale_num_rows;
|
||||
const int64_t col_idx = group_id % scale_num_rows;
|
||||
output_s[col_idx * scale_stride + row_idx] = y_s;
|
||||
} else {
|
||||
output_s[group_id] = y_s;
|
||||
}
|
||||
}
|
||||
|
||||
// A UE8M0 scale is an exact power of two, so multiplying by the reciprocal
|
||||
// matches dividing. Otherwise the reciprocal costs an ulp and can move
|
||||
// values that land exactly on an 8-bit boundary, so divide instead to stay
|
||||
// bit-exact with the smem kernel.
|
||||
const float inv_y = 1.0f / y_s;
|
||||
uint32_t packed[VEC_SIZE / 4] = {};
|
||||
#pragma unroll
|
||||
for (int i = 0; i < VEC_SIZE; ++i) {
|
||||
const float v = static_cast<float>(regs[i]);
|
||||
const float scaled = SCALE_UE8M0 ? (v * inv_y) : (v / y_s);
|
||||
const float q = fminf(fmaxf(scaled, min_8bit), max_8bit);
|
||||
DST_DTYPE qb = DST_DTYPE(q);
|
||||
const uint8_t byte = *reinterpret_cast<uint8_t*>(&qb);
|
||||
packed[i >> 2] |= static_cast<uint32_t>(byte) << ((i & 3) * 8);
|
||||
}
|
||||
|
||||
DST_DTYPE* group_output = static_cast<DST_DTYPE*>(output_q) +
|
||||
group_id * GROUP_SIZE + lane_id * VEC_SIZE;
|
||||
*reinterpret_cast<uint4*>(group_output) =
|
||||
make_uint4(packed[0], packed[1], packed[2], packed[3]);
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
// The register path needs group_size 128 with 16-bit input so that one group is
|
||||
// exactly THREADS_PER_GROUP(8) x VEC_SIZE(16) elements, and it loads/stores
|
||||
// through uint4, so both buffers must be 16 B aligned.
|
||||
inline bool CanUseRegisterPath(const torch::stable::Tensor& input,
|
||||
const torch::stable::Tensor& output_q,
|
||||
int64_t group_size) {
|
||||
const auto dtype = input.scalar_type();
|
||||
if (group_size != 128) return false;
|
||||
if (dtype != torch::headeronly::ScalarType::Half &&
|
||||
dtype != torch::headeronly::ScalarType::BFloat16) {
|
||||
return false;
|
||||
}
|
||||
if (!input.is_contiguous() || !output_q.is_contiguous()) return false;
|
||||
const auto in_addr = reinterpret_cast<uintptr_t>(input.data_ptr());
|
||||
const auto out_addr = reinterpret_cast<uintptr_t>(output_q.data_ptr());
|
||||
return (in_addr % 16 == 0) && (out_addr % 16 == 0);
|
||||
}
|
||||
|
||||
inline int GetGroupsPerBlock(int64_t num_groups) {
|
||||
if (num_groups % 16 == 0) {
|
||||
return 16;
|
||||
}
|
||||
if (num_groups % 8 == 0) {
|
||||
return 8;
|
||||
}
|
||||
if (num_groups % 4 == 0) {
|
||||
return 4;
|
||||
}
|
||||
if (num_groups % 2 == 0) {
|
||||
return 2;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Largest divisor of padded_groups_per_row that is <= 16. ry = 16 / kx.
|
||||
inline int GetGroupsPerBlockX(int64_t padded_groups_per_row) {
|
||||
if (padded_groups_per_row % 16 == 0) {
|
||||
return 16;
|
||||
}
|
||||
if (padded_groups_per_row % 8 == 0) {
|
||||
return 8;
|
||||
}
|
||||
return 4;
|
||||
}
|
||||
|
||||
// 8-bit per-token-group quantization helper used by both FP8 and INT8
|
||||
static void per_token_group_quant_8bit(const torch::stable::Tensor& input,
|
||||
torch::stable::Tensor& output_q,
|
||||
torch::stable::Tensor& output_s,
|
||||
int64_t group_size, double eps,
|
||||
double min_8bit, double max_8bit,
|
||||
bool scale_ue8m0 = false) {
|
||||
STD_TORCH_CHECK(input.is_contiguous());
|
||||
STD_TORCH_CHECK(output_q.is_contiguous());
|
||||
|
||||
const int num_groups = input.numel() / group_size;
|
||||
|
||||
STD_TORCH_CHECK(input.numel() % group_size == 0);
|
||||
STD_TORCH_CHECK(output_s.dim() == 2);
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
cudaStream_t stream = get_current_cuda_stream();
|
||||
|
||||
constexpr int THREADS_PER_GROUP = 16;
|
||||
|
||||
if (num_groups == 0) {
|
||||
return;
|
||||
}
|
||||
STD_TORCH_CHECK(input.size(-1) % group_size == 0);
|
||||
const int64_t groups_per_row = input.size(-1) / group_size;
|
||||
const int64_t num_rows = num_groups / groups_per_row;
|
||||
// Odd row counts inherit the next even row count's launch geometry.
|
||||
const int64_t policy_num_groups =
|
||||
(num_rows + (num_rows & 1)) * groups_per_row;
|
||||
const int groups_per_block = GetGroupsPerBlock(policy_num_groups);
|
||||
const bool needs_bounds_check = num_groups % groups_per_block != 0;
|
||||
|
||||
auto dst_type = output_q.scalar_type();
|
||||
const int num_blocks = (num_groups + groups_per_block - 1) / groups_per_block;
|
||||
const int num_threads = groups_per_block * THREADS_PER_GROUP;
|
||||
|
||||
const bool is_column_major = output_s.stride(0) < output_s.stride(1);
|
||||
const int scale_num_rows = output_s.size(1);
|
||||
const int scale_stride = output_s.stride(1);
|
||||
|
||||
const bool use_register_path =
|
||||
CanUseRegisterPath(input, output_q, group_size);
|
||||
constexpr int REG_THREADS_PER_GROUP = 8;
|
||||
constexpr int REG_BLOCK_THREADS = 256;
|
||||
const int64_t reg_groups_per_block =
|
||||
REG_BLOCK_THREADS / REG_THREADS_PER_GROUP;
|
||||
const int reg_num_blocks = static_cast<int>(
|
||||
(num_groups + reg_groups_per_block - 1) / reg_groups_per_block);
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#define LAUNCH_KERNEL_IMPL(T, DST_DTYPE, COL_MAJOR, UE8M0, CHECK_BOUNDS, \
|
||||
SMEM_BYTES) \
|
||||
do { \
|
||||
cudaLaunchConfig_t config = {}; \
|
||||
config.gridDim = dim3(num_blocks); \
|
||||
config.blockDim = dim3(num_threads); \
|
||||
config.dynamicSmemBytes = (SMEM_BYTES); \
|
||||
config.stream = stream; \
|
||||
cudaLaunchAttribute attrs[1]; \
|
||||
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; \
|
||||
attrs[0].val.programmaticStreamSerializationAllowed = 1; \
|
||||
config.numAttrs = 1; \
|
||||
config.attrs = attrs; \
|
||||
cudaLaunchKernelEx( \
|
||||
&config, \
|
||||
per_token_group_quant_8bit_kernel<T, DST_DTYPE, COL_MAJOR, UE8M0, \
|
||||
float, CHECK_BOUNDS>, \
|
||||
static_cast<T*>(input.data_ptr()), output_q.data_ptr(), \
|
||||
static_cast<float*>(output_s.data_ptr()), group_size, num_groups, \
|
||||
groups_per_block, (float)eps, (float)min_8bit, (float)max_8bit, \
|
||||
scale_num_rows, scale_stride); \
|
||||
} while (0)
|
||||
#else
|
||||
#define LAUNCH_KERNEL_IMPL(T, DST_DTYPE, COL_MAJOR, UE8M0, CHECK_BOUNDS, \
|
||||
SMEM_BYTES) \
|
||||
do { \
|
||||
per_token_group_quant_8bit_kernel<T, DST_DTYPE, COL_MAJOR, UE8M0, float, \
|
||||
CHECK_BOUNDS> \
|
||||
<<<dim3(num_blocks), dim3(num_threads), (SMEM_BYTES), stream>>>( \
|
||||
static_cast<T*>(input.data_ptr()), output_q.data_ptr(), \
|
||||
static_cast<float*>(output_s.data_ptr()), group_size, \
|
||||
num_groups, groups_per_block, (float)eps, (float)min_8bit, \
|
||||
(float)max_8bit, scale_num_rows, scale_stride); \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
#define LAUNCH_KERNEL_INST(T, DST_DTYPE, COL_MAJOR, UE8M0, SMEM_BYTES) \
|
||||
do { \
|
||||
if (needs_bounds_check) { \
|
||||
LAUNCH_KERNEL_IMPL(T, DST_DTYPE, COL_MAJOR, UE8M0, true, SMEM_BYTES); \
|
||||
} else { \
|
||||
LAUNCH_KERNEL_IMPL(T, DST_DTYPE, COL_MAJOR, UE8M0, false, SMEM_BYTES); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#define LAUNCH_REGISTER_INST(T, DST_DTYPE, COL_MAJOR, UE8M0) \
|
||||
do { \
|
||||
cudaLaunchConfig_t config = {}; \
|
||||
config.gridDim = dim3(reg_num_blocks); \
|
||||
config.blockDim = dim3(REG_BLOCK_THREADS); \
|
||||
config.dynamicSmemBytes = 0; \
|
||||
config.stream = stream; \
|
||||
cudaLaunchAttribute attrs[1]; \
|
||||
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; \
|
||||
attrs[0].val.programmaticStreamSerializationAllowed = 1; \
|
||||
config.numAttrs = 1; \
|
||||
config.attrs = attrs; \
|
||||
cudaLaunchKernelEx( \
|
||||
&config, \
|
||||
per_token_group_quant_8bit_register_kernel<T, DST_DTYPE, COL_MAJOR, \
|
||||
UE8M0>, \
|
||||
static_cast<T*>(input.data_ptr()), output_q.data_ptr(), \
|
||||
static_cast<float*>(output_s.data_ptr()), \
|
||||
static_cast<int64_t>(num_groups), (float)eps, (float)min_8bit, \
|
||||
(float)max_8bit, scale_num_rows, scale_stride); \
|
||||
} while (0)
|
||||
#else
|
||||
#define LAUNCH_REGISTER_INST(T, DST_DTYPE, COL_MAJOR, UE8M0) \
|
||||
do { \
|
||||
per_token_group_quant_8bit_register_kernel<T, DST_DTYPE, COL_MAJOR, \
|
||||
UE8M0> \
|
||||
<<<dim3(reg_num_blocks), dim3(REG_BLOCK_THREADS), 0, stream>>>( \
|
||||
static_cast<T*>(input.data_ptr()), output_q.data_ptr(), \
|
||||
static_cast<float*>(output_s.data_ptr()), \
|
||||
static_cast<int64_t>(num_groups), (float)eps, (float)min_8bit, \
|
||||
(float)max_8bit, scale_num_rows, scale_stride); \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
#define LAUNCH_KERNEL(T, DST_DTYPE) \
|
||||
do { \
|
||||
size_t smem_bytes = \
|
||||
static_cast<size_t>(groups_per_block) * group_size * sizeof(T); \
|
||||
if (is_column_major) { \
|
||||
if (scale_ue8m0) { \
|
||||
if (use_register_path) { \
|
||||
LAUNCH_REGISTER_INST(T, DST_DTYPE, true, true); \
|
||||
} else { \
|
||||
LAUNCH_KERNEL_INST(T, DST_DTYPE, true, true, smem_bytes); \
|
||||
} \
|
||||
} else { \
|
||||
if (use_register_path) { \
|
||||
LAUNCH_REGISTER_INST(T, DST_DTYPE, true, false); \
|
||||
} else { \
|
||||
LAUNCH_KERNEL_INST(T, DST_DTYPE, true, false, smem_bytes); \
|
||||
} \
|
||||
} \
|
||||
} else { \
|
||||
if (scale_ue8m0) { \
|
||||
if (use_register_path) { \
|
||||
LAUNCH_REGISTER_INST(T, DST_DTYPE, false, true); \
|
||||
} else { \
|
||||
LAUNCH_KERNEL_INST(T, DST_DTYPE, false, true, smem_bytes); \
|
||||
} \
|
||||
} else { \
|
||||
if (use_register_path) { \
|
||||
LAUNCH_REGISTER_INST(T, DST_DTYPE, false, false); \
|
||||
} else { \
|
||||
LAUNCH_KERNEL_INST(T, DST_DTYPE, false, false, smem_bytes); \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "per_token_group_quant_8bit", ([&] {
|
||||
if (dst_type == torch::headeronly::ScalarType::Char) {
|
||||
LAUNCH_KERNEL(scalar_t, int8_t);
|
||||
} else {
|
||||
VLLM_STABLE_DISPATCH_FP8_TYPES(
|
||||
dst_type, "per_token_group_quant_8bit_fp8",
|
||||
([&] { LAUNCH_KERNEL(scalar_t, fp8_t); }));
|
||||
}
|
||||
}));
|
||||
|
||||
#undef LAUNCH_KERNEL
|
||||
#undef LAUNCH_KERNEL_INST
|
||||
#undef LAUNCH_KERNEL_IMPL
|
||||
#undef LAUNCH_REGISTER_INST
|
||||
}
|
||||
|
||||
// Register-resident fast path for group_size==128.
|
||||
//
|
||||
// Each thread holds 16 source elements (32 B = uint4 x 2) in registers across
|
||||
// the absmax reduce -> scale compute -> quantize pipeline. No shared memory.
|
||||
// UE8M0 scale extracted via bit math (bit-exact with exp2f(ceilf(log2f))).
|
||||
//
|
||||
// Loads two contiguous uint4s (16 B + 16 B = 32 B) per thread; on Blackwell
|
||||
// nvcc fuses these into a single 256-bit LDG.E.256.
|
||||
//
|
||||
// Each thread quantizes 16 bf16/fp16 values, using 2 or 8 threads per group.
|
||||
template <typename T, typename DST_DTYPE, int GROUP_SIZE, int kGroupsPerBlockX,
|
||||
int kRowsPerBlock>
|
||||
__global__ void per_token_group_quant_8bit_packed_register_kernel(
|
||||
const T* __restrict__ input, void* __restrict__ output_q,
|
||||
unsigned int* __restrict__ output_s_packed, const int padded_groups_per_row,
|
||||
const int groups_per_row, const int mn, const int output_q_mn_extent,
|
||||
const int tma_aligned_mn, const int64_t num_scale_elems, const float eps,
|
||||
const float min_8bit, const float max_8bit) {
|
||||
static_assert(GROUP_SIZE == 32 || GROUP_SIZE == 128);
|
||||
constexpr int VEC_SIZE = 32 / sizeof(T); // 16 for bf16/fp16
|
||||
constexpr int THREADS_PER_GROUP = GROUP_SIZE / VEC_SIZE;
|
||||
static_assert(GROUP_SIZE == THREADS_PER_GROUP * VEC_SIZE,
|
||||
"GROUP_SIZE must equal THREADS_PER_GROUP * VEC_SIZE");
|
||||
static_assert(32 % THREADS_PER_GROUP == 0,
|
||||
"THREADS_PER_GROUP must divide warp size for the shuffle "
|
||||
"mask to be valid");
|
||||
static_assert(
|
||||
kGroupsPerBlockX > 0 && (kGroupsPerBlockX & (kGroupsPerBlockX - 1)) == 0,
|
||||
"kGroupsPerBlockX must be a positive power of 2");
|
||||
static_assert(kRowsPerBlock > 0, "kRowsPerBlock must be positive");
|
||||
|
||||
const int local_group_id = threadIdx.x / THREADS_PER_GROUP;
|
||||
const int lane_id = threadIdx.x % THREADS_PER_GROUP;
|
||||
|
||||
const int sf_k_local = local_group_id % kGroupsPerBlockX;
|
||||
const int row_local = local_group_id / kGroupsPerBlockX;
|
||||
// Rows on grid.x: mn scales with tokens and can exceed the 65535 grid.y cap.
|
||||
const int sf_k_idx = blockIdx.y * kGroupsPerBlockX + sf_k_local;
|
||||
const int mn_idx = blockIdx.x * kRowsPerBlock + row_local;
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
|
||||
if (mn_idx >= tma_aligned_mn) {
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
|
||||
const bool is_valid_group = (mn_idx < mn) && (sf_k_idx < groups_per_row);
|
||||
|
||||
// Load 16 input elements (32 B) into registers as two adjacent uint4
|
||||
// loads. nvcc keeps these as 2x LDG.E.128 on sm_100; the per-thread cost
|
||||
// is dominated by HBM bandwidth at large MN, so a fused 256-bit load via
|
||||
// inline PTX gave no measurable speedup.
|
||||
// alignas(16) is required so the uint4* reinterpret_cast below is
|
||||
// well-defined for T == bf16/fp16 (default alignof is 2).
|
||||
alignas(16) T regs[VEC_SIZE];
|
||||
float local_absmax = eps;
|
||||
if (is_valid_group) {
|
||||
const T* group_input =
|
||||
input + static_cast<int64_t>(mn_idx) * groups_per_row * GROUP_SIZE +
|
||||
sf_k_idx * GROUP_SIZE + lane_id * VEC_SIZE;
|
||||
uint4* dst = reinterpret_cast<uint4*>(®s[0]);
|
||||
const uint4* src = reinterpret_cast<const uint4*>(group_input);
|
||||
dst[0] = src[0];
|
||||
dst[1] = src[1];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < VEC_SIZE; ++i) {
|
||||
float v = fabsf(static_cast<float>(regs[i]));
|
||||
local_absmax = fmaxf(local_absmax, v);
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef USE_ROCM
|
||||
const int lane_in_wave = threadIdx.x % warpSize;
|
||||
const unsigned long long mask = ((1ull << THREADS_PER_GROUP) - 1)
|
||||
<< (lane_in_wave & ~(THREADS_PER_GROUP - 1));
|
||||
#else
|
||||
const unsigned mask = ((1u << THREADS_PER_GROUP) - 1)
|
||||
<< ((threadIdx.x % 32) & ~(THREADS_PER_GROUP - 1));
|
||||
#endif
|
||||
#pragma unroll
|
||||
for (int offset = THREADS_PER_GROUP / 2; offset > 0; offset /= 2) {
|
||||
local_absmax =
|
||||
fmaxf(local_absmax,
|
||||
__shfl_xor_sync(mask, local_absmax, offset, THREADS_PER_GROUP));
|
||||
}
|
||||
|
||||
float y_s = local_absmax / max_8bit;
|
||||
y_s = fmaxf(y_s, 1e-10f);
|
||||
uint32_t bits = __float_as_uint(y_s);
|
||||
uint32_t exp_bits = (bits >> 23) & 0xffu;
|
||||
uint32_t mant_bits = bits & 0x7fffffu;
|
||||
uint8_t exp_byte =
|
||||
static_cast<uint8_t>(exp_bits + (mant_bits != 0u ? 1u : 0u));
|
||||
|
||||
// Lane 0 writes the packed scale byte.
|
||||
if (lane_id == 0) {
|
||||
const int sf_k_pack_idx = sf_k_idx / 4;
|
||||
const int pos = sf_k_idx % 4;
|
||||
const int out_idx = sf_k_pack_idx * tma_aligned_mn + mn_idx;
|
||||
if (is_valid_group) {
|
||||
reinterpret_cast<uint8_t*>(output_s_packed)[out_idx * 4 + pos] = exp_byte;
|
||||
} else if (out_idx < num_scale_elems) {
|
||||
reinterpret_cast<uint8_t*>(output_s_packed)[out_idx * 4 + pos] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// For padded mn rows that fall within output_q's allocated extent, write
|
||||
// a uint4 of zeros to keep the buffer clean for downstream TMA loads.
|
||||
// Skip writes for sf_k padding (those positions don't exist in output_q).
|
||||
if (!is_valid_group) {
|
||||
if (sf_k_idx < groups_per_row && mn_idx >= mn &&
|
||||
mn_idx < output_q_mn_extent) {
|
||||
DST_DTYPE* group_output =
|
||||
static_cast<DST_DTYPE*>(output_q) +
|
||||
static_cast<int64_t>(mn_idx) * groups_per_row * GROUP_SIZE +
|
||||
sf_k_idx * GROUP_SIZE + lane_id * VEC_SIZE;
|
||||
*reinterpret_cast<uint4*>(group_output) = make_uint4(0, 0, 0, 0);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Reconstruct y_s as a power-of-2 float and use its reciprocal.
|
||||
float y_s_q = __uint_as_float(static_cast<uint32_t>(exp_byte) << 23);
|
||||
float inv_y = 1.0f / y_s_q;
|
||||
|
||||
// Quantize and pack into 16 fp8/int8 bytes (= uint4). VEC_SIZE==16 so we
|
||||
// fill four 32-bit words, four bytes each.
|
||||
uint32_t packed_lo = 0;
|
||||
uint32_t packed_lo_hi = 0;
|
||||
uint32_t packed_hi_lo = 0;
|
||||
uint32_t packed_hi = 0;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < VEC_SIZE; ++i) {
|
||||
float q =
|
||||
fminf(fmaxf(static_cast<float>(regs[i]) * inv_y, min_8bit), max_8bit);
|
||||
DST_DTYPE qb = DST_DTYPE(q);
|
||||
uint8_t byte = *reinterpret_cast<uint8_t*>(&qb);
|
||||
const int shift = (i & 3) * 8;
|
||||
if (i < 4) {
|
||||
packed_lo |= static_cast<uint32_t>(byte) << shift;
|
||||
} else if (i < 8) {
|
||||
packed_lo_hi |= static_cast<uint32_t>(byte) << shift;
|
||||
} else if (i < 12) {
|
||||
packed_hi_lo |= static_cast<uint32_t>(byte) << shift;
|
||||
} else {
|
||||
packed_hi |= static_cast<uint32_t>(byte) << shift;
|
||||
}
|
||||
}
|
||||
|
||||
uint4 packed_out =
|
||||
make_uint4(packed_lo, packed_lo_hi, packed_hi_lo, packed_hi);
|
||||
DST_DTYPE* group_output =
|
||||
static_cast<DST_DTYPE*>(output_q) +
|
||||
static_cast<int64_t>(mn_idx) * groups_per_row * GROUP_SIZE +
|
||||
sf_k_idx * GROUP_SIZE + lane_id * VEC_SIZE;
|
||||
*reinterpret_cast<uint4*>(group_output) = packed_out;
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
}
|
||||
|
||||
// Public entry point: register-resident packed quant kernel.
|
||||
// Constraints: group_size in {32, 128} and bf16/fp16 input.
|
||||
void per_token_group_quant_8bit_packed(const torch::stable::Tensor& input,
|
||||
torch::stable::Tensor& output_q,
|
||||
torch::stable::Tensor& output_s_packed,
|
||||
int64_t group_size, double eps,
|
||||
double min_8bit, double max_8bit) {
|
||||
STD_TORCH_CHECK(group_size == 32 || group_size == 128,
|
||||
"per_token_group_quant_8bit_packed only supports "
|
||||
"group_size in {32, 128}, got ",
|
||||
group_size, ".");
|
||||
const auto in_dtype = input.scalar_type();
|
||||
STD_TORCH_CHECK(
|
||||
in_dtype == torch::headeronly::ScalarType::Half ||
|
||||
in_dtype == torch::headeronly::ScalarType::BFloat16,
|
||||
"per_token_group_quant_8bit_packed only supports bf16/fp16 input.");
|
||||
|
||||
STD_TORCH_CHECK(input.is_contiguous());
|
||||
STD_TORCH_CHECK(output_q.is_contiguous());
|
||||
|
||||
const int64_t k = input.size(-1);
|
||||
STD_TORCH_CHECK(k % group_size == 0, "input last dim k=", k,
|
||||
" is not divisible by group_size=", group_size, ".");
|
||||
|
||||
const int64_t mn = input.numel() / k;
|
||||
const int64_t groups_per_row = k / group_size;
|
||||
const int64_t k_num_packed_sfk = (groups_per_row + 3) / 4;
|
||||
const int64_t tma_aligned_mn = ((mn + 3) / 4) * 4;
|
||||
|
||||
// output_q may be allocated with extra padded mn rows (e.g.,
|
||||
// (tma_aligned_mn, k)) so the kernel can zero-fill them in-line and the
|
||||
// caller can use torch.empty instead of torch.zeros. The grid only covers
|
||||
// up to tma_aligned_mn, so we cap the extent there.
|
||||
const int64_t output_q_mn_actual = output_q.numel() / k;
|
||||
STD_TORCH_CHECK(output_q_mn_actual >= mn,
|
||||
"output_q must have at least mn rows; got ",
|
||||
output_q_mn_actual, " rows for mn=", mn, ".");
|
||||
const int64_t output_q_mn_extent =
|
||||
output_q_mn_actual < tma_aligned_mn ? output_q_mn_actual : tma_aligned_mn;
|
||||
|
||||
STD_TORCH_CHECK(
|
||||
output_s_packed.scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"output_s_packed must be int32 for UE8M0-packed scales.");
|
||||
STD_TORCH_CHECK(output_s_packed.size(0) == mn &&
|
||||
output_s_packed.size(1) == k_num_packed_sfk,
|
||||
"output_s_packed shape must be [", mn, ", ", k_num_packed_sfk,
|
||||
"]; got [", output_s_packed.size(0), ", ",
|
||||
output_s_packed.size(1), "].");
|
||||
STD_TORCH_CHECK(output_s_packed.stride(0) == 1 &&
|
||||
output_s_packed.stride(1) == tma_aligned_mn,
|
||||
"output_s_packed strides must be [1, ", tma_aligned_mn,
|
||||
"]; got [", output_s_packed.stride(0), ", ",
|
||||
output_s_packed.stride(1), "].");
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
cudaStream_t stream = get_current_cuda_stream();
|
||||
|
||||
const int threads_per_group = group_size / 16;
|
||||
const int64_t padded_groups_per_row = k_num_packed_sfk * 4;
|
||||
const int64_t num_scale_elems = mn + (k_num_packed_sfk - 1) * tma_aligned_mn;
|
||||
|
||||
STD_TORCH_CHECK(padded_groups_per_row % 4 == 0,
|
||||
"padded_groups_per_row=", padded_groups_per_row,
|
||||
" is not a multiple of 4.");
|
||||
const int kx = GetGroupsPerBlockX(padded_groups_per_row);
|
||||
const int ry = 16 / kx;
|
||||
const int64_t row_blocks = (tma_aligned_mn + ry - 1) / ry;
|
||||
const int64_t sf_k_blocks = padded_groups_per_row / kx;
|
||||
const int num_threads = (kx * ry) * threads_per_group;
|
||||
// CUDA caps grid.x at 2^31 - 1 and grid.y at 2^16 - 1 (65535).
|
||||
constexpr int64_t kMaxGridDimYZ = 65535;
|
||||
STD_TORCH_CHECK(row_blocks <= static_cast<int64_t>(INT32_MAX) &&
|
||||
sf_k_blocks <= kMaxGridDimYZ,
|
||||
"per_token_group_quant_8bit_packed grid too large: (",
|
||||
row_blocks, ", ", sf_k_blocks, ").");
|
||||
|
||||
auto dst_type = output_q.scalar_type();
|
||||
|
||||
// PDL (Programmatic Dependent Launch) is NVIDIA-only; ROCm/HIP has no
|
||||
// equivalent launch attribute, so fall back to a classic launch there.
|
||||
#ifndef USE_ROCM
|
||||
#define LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, GROUP_SIZE, KX, RY) \
|
||||
do { \
|
||||
cudaLaunchConfig_t config = {}; \
|
||||
config.gridDim = dim3(static_cast<unsigned int>(row_blocks), \
|
||||
static_cast<unsigned int>(sf_k_blocks)); \
|
||||
config.blockDim = dim3(num_threads); \
|
||||
config.dynamicSmemBytes = 0; \
|
||||
config.stream = stream; \
|
||||
cudaLaunchAttribute attrs[1]; \
|
||||
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; \
|
||||
attrs[0].val.programmaticStreamSerializationAllowed = 1; \
|
||||
config.numAttrs = 1; \
|
||||
config.attrs = attrs; \
|
||||
cudaLaunchKernelEx( \
|
||||
&config, \
|
||||
per_token_group_quant_8bit_packed_register_kernel< \
|
||||
T, DST_DTYPE, GROUP_SIZE, KX, RY>, \
|
||||
static_cast<const T*>(input.data_ptr()), output_q.data_ptr(), \
|
||||
reinterpret_cast<unsigned int*>(output_s_packed.data_ptr()), \
|
||||
static_cast<int>(padded_groups_per_row), \
|
||||
static_cast<int>(groups_per_row), static_cast<int>(mn), \
|
||||
static_cast<int>(output_q_mn_extent), \
|
||||
static_cast<int>(tma_aligned_mn), num_scale_elems, \
|
||||
static_cast<float>(eps), static_cast<float>(min_8bit), \
|
||||
static_cast<float>(max_8bit)); \
|
||||
} while (0)
|
||||
#else
|
||||
#define LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, GROUP_SIZE, KX, RY) \
|
||||
do { \
|
||||
dim3 grid(static_cast<unsigned int>(row_blocks), \
|
||||
static_cast<unsigned int>(sf_k_blocks)); \
|
||||
dim3 block(num_threads); \
|
||||
per_token_group_quant_8bit_packed_register_kernel<T, DST_DTYPE, \
|
||||
GROUP_SIZE, KX, RY> \
|
||||
<<<grid, block, 0, stream>>>( \
|
||||
static_cast<const T*>(input.data_ptr()), output_q.data_ptr(), \
|
||||
reinterpret_cast<unsigned int*>(output_s_packed.data_ptr()), \
|
||||
static_cast<int>(padded_groups_per_row), \
|
||||
static_cast<int>(groups_per_row), static_cast<int>(mn), \
|
||||
static_cast<int>(output_q_mn_extent), \
|
||||
static_cast<int>(tma_aligned_mn), num_scale_elems, \
|
||||
static_cast<float>(eps), static_cast<float>(min_8bit), \
|
||||
static_cast<float>(max_8bit)); \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
#define LAUNCH_REG_KERNEL_GROUP_SIZE(T, DST_DTYPE, GROUP_SIZE) \
|
||||
do { \
|
||||
if (kx == 16) { \
|
||||
LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, GROUP_SIZE, 16, 1); \
|
||||
} else if (kx == 8) { \
|
||||
LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, GROUP_SIZE, 8, 2); \
|
||||
} else if (kx == 4) { \
|
||||
LAUNCH_REG_KERNEL_INST(T, DST_DTYPE, GROUP_SIZE, 4, 4); \
|
||||
} else { \
|
||||
STD_TORCH_CHECK(false, "Unsupported kx value ", kx); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define LAUNCH_REG_KERNEL(T, DST_DTYPE) \
|
||||
do { \
|
||||
if (group_size == 32) { \
|
||||
LAUNCH_REG_KERNEL_GROUP_SIZE(T, DST_DTYPE, 32); \
|
||||
} else { \
|
||||
LAUNCH_REG_KERNEL_GROUP_SIZE(T, DST_DTYPE, 128); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
VLLM_STABLE_DISPATCH_HALF_TYPES(
|
||||
input.scalar_type(), "per_token_group_quant_8bit_packed_register", ([&] {
|
||||
if (dst_type == torch::headeronly::ScalarType::Char) {
|
||||
LAUNCH_REG_KERNEL(scalar_t, int8_t);
|
||||
} else {
|
||||
VLLM_STABLE_DISPATCH_FP8_TYPES(
|
||||
dst_type, "per_token_group_quant_8bit_packed_fp8",
|
||||
([&] { LAUNCH_REG_KERNEL(scalar_t, fp8_t); }));
|
||||
}
|
||||
}));
|
||||
|
||||
#undef LAUNCH_REG_KERNEL
|
||||
#undef LAUNCH_REG_KERNEL_GROUP_SIZE
|
||||
#undef LAUNCH_REG_KERNEL_INST
|
||||
}
|
||||
|
||||
void per_token_group_quant_fp8(const torch::stable::Tensor& input,
|
||||
torch::stable::Tensor& output_q,
|
||||
torch::stable::Tensor& output_s,
|
||||
int64_t group_size, double eps, double fp8_min,
|
||||
double fp8_max, bool scale_ue8m0,
|
||||
bool dummy_is_scale_transposed = false,
|
||||
bool dummy_is_tma_aligned = false) {
|
||||
per_token_group_quant_8bit(input, output_q, output_s, group_size, eps,
|
||||
fp8_min, fp8_max, scale_ue8m0);
|
||||
}
|
||||
|
||||
void per_token_group_quant_int8(const torch::stable::Tensor& input,
|
||||
torch::stable::Tensor& output_q,
|
||||
torch::stable::Tensor& output_s,
|
||||
int64_t group_size, double eps, double int8_min,
|
||||
double int8_max) {
|
||||
per_token_group_quant_8bit(input, output_q, output_s, group_size, eps,
|
||||
int8_min, int8_max);
|
||||
}
|
||||
Reference in New Issue
Block a user