New stable abi cleanup (#46656)
Signed-off-by: Chris Leonard <chleonar@redhat.com>
This commit is contained in:
@@ -733,6 +733,8 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output,
|
||||
output.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||
"Only BFloat16 output dtype is supported");
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
mat_a.get_device_index());
|
||||
STD_TORCH_CHECK(getSMVersion() >= 90, "required CUDA ARCH >= SM_90");
|
||||
|
||||
auto stream = get_current_cuda_stream(mat_a.get_device_index());
|
||||
|
||||
@@ -127,6 +127,8 @@ void fp32_router_gemm(
|
||||
return;
|
||||
}
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
mat_a.get_device_index());
|
||||
STD_TORCH_CHECK(getSMVersion() >= 90, "fp32_router_gemm: requires SM90+");
|
||||
|
||||
auto stream = get_current_cuda_stream(mat_a.get_device_index());
|
||||
|
||||
@@ -57,7 +57,7 @@
|
||||
#include "torch_utils.h"
|
||||
|
||||
#include "../cuda_compat.h"
|
||||
#include "../type_convert.cuh"
|
||||
#include "type_convert.cuh"
|
||||
#include "../attention/dtype_fp8.cuh"
|
||||
#include "dispatch_utils.h"
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
|
||||
#include "async_util.cuh"
|
||||
#include "../cuda_compat.h"
|
||||
#include "../type_convert.cuh"
|
||||
#include "type_convert.cuh"
|
||||
#include "dispatch_utils.h"
|
||||
|
||||
#define CHECK_TYPE(x, st) \
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
#include "torch_utils.h"
|
||||
|
||||
#include "../cub_helpers.h"
|
||||
#include "cub_helpers.h"
|
||||
#include "../core/batch_invariant.hpp"
|
||||
#include "../type_convert.cuh"
|
||||
#include "type_convert.cuh"
|
||||
#include "dispatch_utils.h"
|
||||
#include "quantization/vectorization_utils.cuh"
|
||||
|
||||
|
||||
@@ -9,10 +9,10 @@
|
||||
|
||||
#include "torch_utils.h"
|
||||
|
||||
#include "../cub_helpers.h"
|
||||
#include "cub_helpers.h"
|
||||
#include "../core/batch_invariant.hpp"
|
||||
#include "../quantization/w8a8/fp8/common.cuh"
|
||||
#include "../type_convert.cuh"
|
||||
#include "type_convert.cuh"
|
||||
#include "dispatch_utils.h"
|
||||
#include "quantization/vectorization_utils.cuh"
|
||||
|
||||
|
||||
@@ -151,6 +151,8 @@ void dsv3_router_gemm(
|
||||
output.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||
"output must be float32 or bf16");
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
mat_a.get_device_index());
|
||||
const int sm = getSMVersion();
|
||||
STD_TORCH_CHECK(sm >= 90, "required CUDA ARCH >= SM_90");
|
||||
|
||||
|
||||
@@ -1040,6 +1040,9 @@ std::tuple<torch::stable::Tensor, torch::stable::Tensor> grouped_topk(
|
||||
auto topk_indices = torch::stable::new_empty(
|
||||
scores, {num_tokens, topk}, torch::headeronly::ScalarType::Int);
|
||||
const bool pdl_flag = num_tokens <= vllm::moe::PDLEnableTokens;
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
scores.get_device_index());
|
||||
const cudaStream_t stream =
|
||||
get_current_cuda_stream(scores.get_device_index());
|
||||
auto const sf = static_cast<vllm::moe::ScoringFunc>(scoring_func);
|
||||
|
||||
@@ -584,6 +584,8 @@ void moe_align_block_size(
|
||||
torch::stable::Tensor sorted_token_ids, torch::stable::Tensor experts_ids,
|
||||
torch::stable::Tensor num_tokens_post_pad,
|
||||
std::optional<torch::stable::Tensor> maybe_expert_map) {
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
topk_ids.get_device_index());
|
||||
const cudaStream_t stream =
|
||||
get_current_cuda_stream(topk_ids.get_device_index());
|
||||
|
||||
@@ -685,6 +687,8 @@ void batched_moe_align_block_size(int64_t max_tokens_per_batch,
|
||||
torch::stable::Tensor num_tokens_post_pad) {
|
||||
namespace batched_kernel = vllm::moe::batched_moe_align_block_size;
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
batch_num_tokens.get_device_index());
|
||||
const cudaStream_t stream =
|
||||
get_current_cuda_stream(batch_num_tokens.get_device_index());
|
||||
int32_t const B = batch_num_tokens.size(0);
|
||||
@@ -802,6 +806,7 @@ void moe_lora_align_block_size(
|
||||
|
||||
int device_max_shared_mem;
|
||||
int dev = topk_ids.get_device_index();
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(dev);
|
||||
cudaDeviceGetAttribute(&device_max_shared_mem,
|
||||
cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
|
||||
const cudaStream_t stream = get_current_cuda_stream(dev);
|
||||
|
||||
@@ -87,6 +87,8 @@ void moe_permute_impl(
|
||||
inv_permuted_idx.sizes().equals(token_expert_indices.sizes()),
|
||||
"token_expert_indices shape must be same as inv_permuted_idx");
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
auto device = input.device();
|
||||
auto n_token = input.sizes()[0];
|
||||
auto n_hidden = input.sizes()[1];
|
||||
@@ -182,6 +184,8 @@ void moe_unpermute(
|
||||
permuted_hidden_states.scalar_type() == hidden_states.scalar_type(),
|
||||
"permuted_hidden_states dtype must be same as hidden_states");
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
hidden_states.get_device_index());
|
||||
auto n_token = hidden_states.size(0);
|
||||
auto n_hidden = hidden_states.size(1);
|
||||
auto stream = get_current_cuda_stream(hidden_states.get_device_index());
|
||||
@@ -238,6 +242,8 @@ void shuffle_rows(const torch::stable::Tensor& input_tensor,
|
||||
STD_TORCH_CHECK(input_tensor.scalar_type() == output_tensor.scalar_type(),
|
||||
"Input and output tensors must have the same data type");
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
output_tensor.get_device_index());
|
||||
auto stream = get_current_cuda_stream(output_tensor.get_device_index());
|
||||
const int64_t blocks = output_tensor.size(0);
|
||||
const int64_t threads = 256;
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
#include <torch/headeronly/util/Exception.h>
|
||||
|
||||
#include "../../cuda_compat.h"
|
||||
#include "../../cub_helpers.h"
|
||||
#include "../cub_helpers.h"
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
|
||||
#ifndef USE_ROCM
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
#include <torch/headeronly/util/Exception.h>
|
||||
|
||||
#include "../../cuda_compat.h"
|
||||
#include "../../cub_helpers.h"
|
||||
#include "../cub_helpers.h"
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
#ifndef USE_ROCM
|
||||
#include <cuda_bf16.h>
|
||||
|
||||
@@ -100,6 +100,8 @@ void run_get_group_gemm_starts(
|
||||
int64_t k = a_tensors.size(1);
|
||||
int64_t scale_k = cutlass::ceil_div(k, b_group_size);
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
a_tensors.get_device_index());
|
||||
auto stream = get_current_cuda_stream(a_tensors.get_device_index());
|
||||
|
||||
if (false) {
|
||||
|
||||
@@ -17,11 +17,11 @@
|
||||
#include <torch/csrc/stable/library.h>
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
#include "cutlass_extensions/torch_utils.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/torch_utils.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/common.hpp"
|
||||
|
||||
#include "get_group_starts.cuh"
|
||||
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
#include "w4a8_utils.cuh"
|
||||
|
||||
namespace vllm::cutlass_w4a8_moe {
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
#include <torch/csrc/stable/library.h>
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
#include "cutlass_extensions/torch_utils.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/torch_utils.hpp"
|
||||
#include "w4a8_utils.cuh"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
@@ -22,7 +22,7 @@
|
||||
#include "cutlass/util/mixed_dtype_utils.hpp"
|
||||
|
||||
#include "libtorch_stable/cutlass_extensions/common.hpp"
|
||||
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
|
||||
@@ -142,6 +142,8 @@ void mxfp4_run_get_group_gemm_starts(
|
||||
torch::stable::Tensor const& sf_offsets,
|
||||
torch::stable::Tensor const& problem_sizes, int M, int N, int K) {
|
||||
int num_experts = (int)expert_offsets.size(0);
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
a_tensors.get_device_index());
|
||||
auto stream = get_current_cuda_stream(a_tensors.get_device_index());
|
||||
|
||||
STD_TORCH_CHECK(out_tensors.size(1) == N,
|
||||
@@ -172,6 +174,8 @@ void run_mxfp4_blockwise_scaled_group_mm_sm100(
|
||||
const torch::stable::Tensor& problem_sizes,
|
||||
const torch::stable::Tensor& expert_offsets,
|
||||
const torch::stable::Tensor& sf_offsets, int M, int N, int K) {
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
a.get_device_index());
|
||||
using ProblemShape =
|
||||
cutlass::gemm::GroupProblemShape<Shape<int32_t, int32_t, int32_t>>;
|
||||
using ElementType = cutlass::float_e2m1_t;
|
||||
|
||||
@@ -173,6 +173,8 @@ void run_get_group_gemm_starts(const torch::stable::Tensor& a_starts,
|
||||
torch::stable::Tensor const& problem_sizes,
|
||||
int M, int N, int K) {
|
||||
int num_experts = (int)expert_offsets.size(0);
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
a_tensors.get_device_index());
|
||||
auto stream = get_current_cuda_stream(a_tensors.get_device_index());
|
||||
|
||||
STD_TORCH_CHECK(out_tensors.size(1) == N,
|
||||
@@ -206,6 +208,8 @@ void run_fp4_blockwise_scaled_group_mm_sm100(
|
||||
const torch::stable::Tensor& problem_sizes,
|
||||
const torch::stable::Tensor& expert_offsets,
|
||||
const torch::stable::Tensor& sf_offsets, int M, int N, int K) {
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
a.get_device_index());
|
||||
using ProblemShape =
|
||||
cutlass::gemm::GroupProblemShape<Shape<int32_t, int32_t, int32_t>>;
|
||||
using ElementType = cutlass::float_e2m1_t;
|
||||
@@ -411,6 +415,8 @@ void run_fp4_blockwise_scaled_group_mm_sm120(
|
||||
const torch::stable::Tensor& problem_sizes,
|
||||
const torch::stable::Tensor& expert_offsets,
|
||||
const torch::stable::Tensor& sf_offsets, int M, int N, int K) {
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
a.get_device_index());
|
||||
using ProblemShape =
|
||||
cutlass::gemm::GroupProblemShape<Shape<int32_t, int32_t, int32_t>>;
|
||||
using ElementType = cutlass::float_e2m1_t;
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
#include "quantization/utils.cuh"
|
||||
#include "quant_conversions.cuh"
|
||||
|
||||
#include "../../../cub_helpers.h"
|
||||
#include "../../cub_helpers.h"
|
||||
#include "../../../cuda_compat.h"
|
||||
|
||||
namespace vllm {
|
||||
|
||||
@@ -150,6 +150,8 @@ void rearrange_kn_weight_as_n32k16_order(
|
||||
void* b_zero_reorder =
|
||||
has_zp ? b_zeros_reorder.value().mutable_data_ptr() : nullptr;
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
b_qweight.get_device_index());
|
||||
cudaStream_t stream = get_current_cuda_stream();
|
||||
if (b_scales.scalar_type() == torch::headeronly::ScalarType::Half) {
|
||||
allspark::rearrange_kn_weight_as_n32k16_order_ldg16<__half>(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include "cutlass_extensions/vllm_collective_builder.cuh"
|
||||
#include "libtorch_stable/cutlass_extensions/vllm_collective_builder.cuh"
|
||||
#include "machete_mainloop.cuh"
|
||||
|
||||
namespace cutlass::gemm::collective {
|
||||
|
||||
@@ -18,9 +18,9 @@
|
||||
// clang-format on
|
||||
|
||||
#include "cutlass_extensions/cute_utils.cuh"
|
||||
#include "cutlass_extensions/vllm_numeric_conversion.cuh"
|
||||
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
#include "cutlass_extensions/torch_utils.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/vllm_numeric_conversion.cuh"
|
||||
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/torch_utils.hpp"
|
||||
#include "machete_collective_builder.cuh"
|
||||
#include "machete_prepacked_layout.cuh"
|
||||
#include "machete_interleaving_utils.cuh"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include "machete_mm_kernel.cuh"
|
||||
#include "cutlass_extensions/torch_utils.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/torch_utils.hpp"
|
||||
#include "core/scalar_type.hpp"
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
#include "machete_mm_kernel.cuh"
|
||||
#include "cutlass_extensions/cute_utils.cuh"
|
||||
#include "cutlass_extensions/torch_utils.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/torch_utils.hpp"
|
||||
#include <torch/headeronly/util/Exception.h>
|
||||
|
||||
namespace machete {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include "machete_prepack_kernel.cuh"
|
||||
#include "cutlass_extensions/torch_utils.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/torch_utils.hpp"
|
||||
#include "core/scalar_type.hpp"
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ void cutlass_gemm_caller(
|
||||
typename GemmKernel::MainloopArguments mainloop_args,
|
||||
typename GemmKernel::EpilogueArguments epilogue_args,
|
||||
typename GemmKernel::TileSchedulerArguments scheduler = {}) {
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(device.index());
|
||||
cutlass::KernelHardwareInfo hw_info;
|
||||
typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
|
||||
prob_shape,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#include "scaled_mm_kernels.hpp"
|
||||
#include "scaled_mm_sm90_int8_dispatch.cuh"
|
||||
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
|
||||
namespace vllm {
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#include "scaled_mm_kernels.hpp"
|
||||
#include "scaled_mm_blockwise_sm100_fp8_dispatch.cuh"
|
||||
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
|
||||
namespace vllm {
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#include "scaled_mm_kernels.hpp"
|
||||
#include "scaled_mm_blockwise_sm120_fp8_dispatch.cuh"
|
||||
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
|
||||
namespace vllm {
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
|
||||
#include "scaled_mm_kernels.hpp"
|
||||
#include "scaled_mm_blockwise_sm90_fp8_dispatch.cuh"
|
||||
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
|
||||
namespace vllm {
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
#include "scaled_mm.cuh"
|
||||
#include "cutlass_gemm_caller.cuh"
|
||||
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
|
||||
/**
|
||||
* This file defines Gemm kernel configurations for SM100 (fp8) based on the
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#include "scaled_mm_kernels.hpp"
|
||||
#include "scaled_mm_sm120_fp8_dispatch.cuh"
|
||||
#include "core/batch_invariant.hpp"
|
||||
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
|
||||
namespace vllm {
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
#include "scaled_mm.cuh"
|
||||
#include "cutlass_gemm_caller.cuh"
|
||||
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
|
||||
/**
|
||||
* This file defines Gemm kernel configurations for SM90 (fp8) based on the Gemm
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#include "scaled_mm_kernels.hpp"
|
||||
#include "scaled_mm_sm90_int8_dispatch.cuh"
|
||||
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
|
||||
namespace vllm {
|
||||
|
||||
|
||||
@@ -74,6 +74,8 @@ void run_get_group_gemm_starts(
|
||||
bool per_act_token = a_scales.numel() != 1;
|
||||
bool per_out_ch = b_scales.numel() != num_experts;
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
a_tensors.get_device_index());
|
||||
auto stream = get_current_cuda_stream(a_tensors.get_device_index());
|
||||
|
||||
if (false) {
|
||||
|
||||
@@ -7,8 +7,9 @@
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
|
||||
#include <torch/csrc/stable/ops.h>
|
||||
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
|
||||
#include "libtorch_stable/cutlass_extensions/common.hpp"
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
#include "get_group_starts.cuh"
|
||||
|
||||
using namespace cute;
|
||||
@@ -103,6 +104,8 @@ void cutlass_group_gemm_caller(torch::stable::Tensor& out_tensors,
|
||||
|
||||
int num_experts = static_cast<int>(expert_offsets.size(0));
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
a_tensors.get_device_index());
|
||||
auto stream = get_current_cuda_stream(a_tensors.get_device_index());
|
||||
|
||||
auto device = a_tensors.device();
|
||||
|
||||
@@ -212,6 +212,8 @@ void get_cutlass_moe_mm_problem_sizes_from_expert_offsets_caller(
|
||||
"n and k must fit in int32");
|
||||
|
||||
int const num_experts = static_cast<int>(num_experts64);
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
expert_first_token_offset.get_device_index());
|
||||
auto stream =
|
||||
get_current_cuda_stream(expert_first_token_offset.get_device_index());
|
||||
|
||||
@@ -241,6 +243,7 @@ void get_cutlass_moe_mm_data_caller(
|
||||
const std::optional<torch::stable::Tensor>& blockscale_offsets,
|
||||
const bool is_gated) {
|
||||
auto device = topk_ids.device();
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(device.index());
|
||||
auto stream = get_current_cuda_stream(device.index());
|
||||
torch::stable::Tensor atomic_buffer = torch::stable::new_zeros(
|
||||
topk_ids, {num_experts}, torch::headeronly::ScalarType::Int);
|
||||
@@ -311,6 +314,8 @@ void get_cutlass_batched_moe_mm_data_caller(
|
||||
const torch::stable::Tensor& expert_num_tokens,
|
||||
const int64_t num_local_experts, const int64_t padded_m, const int64_t n,
|
||||
const int64_t k) {
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
expert_offsets.get_device_index());
|
||||
auto stream = get_current_cuda_stream(expert_offsets.get_device_index());
|
||||
|
||||
if (num_local_experts * padded_m > SWAP_AB_THRESHOLD) {
|
||||
|
||||
@@ -156,6 +156,7 @@ inline void cutlass_gemm_caller(torch::stable::Tensor& out,
|
||||
torch::stable::empty(workspace_size, torch::headeronly::ScalarType::Byte,
|
||||
std::nullopt, device);
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(device.index());
|
||||
auto stream = get_current_cuda_stream(device.index());
|
||||
|
||||
CUTLASS_CHECK(gemm_op.can_implement(args));
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#include "../../../../quantization/w8a8/fp8/common.cuh"
|
||||
#include "../../../dispatch_utils.h"
|
||||
#include "../../../../cub_helpers.h"
|
||||
#include "../../../cub_helpers.h"
|
||||
#include "../../vectorization_utils.cuh"
|
||||
#include "../../../torch_utils.h"
|
||||
#include <torch/csrc/stable/macros.h>
|
||||
|
||||
@@ -203,6 +203,8 @@ void per_token_group_quant_8bit(const torch::stable::Tensor& input,
|
||||
STD_TORCH_CHECK(input.numel() % group_size == 0);
|
||||
STD_TORCH_CHECK(output_s.dim() == 2);
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
cudaStream_t stream = get_current_cuda_stream();
|
||||
|
||||
constexpr int THREADS_PER_GROUP = 16;
|
||||
@@ -506,6 +508,8 @@ void per_token_group_quant_8bit_packed(const torch::stable::Tensor& input,
|
||||
"]; got [", output_s_packed.stride(0), ", ",
|
||||
output_s_packed.stride(1), "].");
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
cudaStream_t stream = get_current_cuda_stream();
|
||||
|
||||
constexpr int THREADS_PER_GROUP = 8;
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
#include "../../../dispatch_utils.h"
|
||||
#include "../../../torch_utils.h"
|
||||
#include "../../vectorization_utils.cuh"
|
||||
#include "../../../../cub_helpers.h"
|
||||
#include "../../../cub_helpers.h"
|
||||
|
||||
static inline __device__ int8_t float_to_int8_rn(float x) {
|
||||
#ifdef USE_ROCM
|
||||
|
||||
@@ -665,6 +665,8 @@ void top_k_per_row_decode(const torch::stable::Tensor& logits, int64_t next_n,
|
||||
constexpr int kSortingAlgorithmThreshold = 12288;
|
||||
constexpr int kSplitWorkThreshold = 200 * 1000;
|
||||
constexpr int kNumThreadsPerBlock = 512;
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
logits.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
const auto numColumns = logits.size(1);
|
||||
|
||||
@@ -727,6 +729,8 @@ void top_k_per_row_prefill(const torch::stable::Tensor& logits,
|
||||
int64_t stride0, int64_t stride1, int64_t topK) {
|
||||
constexpr int kSortingAlgorithmThreshold = 12288;
|
||||
constexpr int kNumThreadsPerBlock = 512;
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
logits.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
|
||||
int numInsertionBlocks =
|
||||
|
||||
@@ -21,6 +21,8 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
|
||||
int64_t max_seq_len) {
|
||||
namespace P = vllm::persistent;
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
logits.get_device_index());
|
||||
const int64_t num_rows = logits.size(0);
|
||||
const int64_t stride = logits.stride(0);
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
@@ -260,6 +262,9 @@ void persistent_topk(const torch::stable::Tensor& logits,
|
||||
k == 512 || k == 1024 || k == 2048,
|
||||
"persistent_topk supports k=512, k=1024, or k=2048, got k=", k);
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
logits.get_device_index());
|
||||
|
||||
if (k == 512) {
|
||||
launch_persistent_topk<512>(logits, lengths, output, workspace,
|
||||
max_seq_len);
|
||||
|
||||
+6
-13
@@ -6,6 +6,7 @@ pynvml. However, it should not initialize cuda context.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import platform
|
||||
from collections.abc import Callable
|
||||
@@ -20,6 +21,9 @@ from typing_extensions import ParamSpec
|
||||
|
||||
# import custom ops, trigger op registration
|
||||
import vllm._C_stable_libtorch # noqa
|
||||
|
||||
with contextlib.suppress(ImportError):
|
||||
import vllm._qutlass_C # noqa
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import import_pynvml
|
||||
@@ -39,11 +43,6 @@ else:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
try:
|
||||
import vllm._qutlass_C # noqa: F401
|
||||
except ImportError as e:
|
||||
logger.warning("Failed to import from vllm._qutlass_C: %r", e)
|
||||
|
||||
_P = ParamSpec("_P")
|
||||
_R = TypeVar("_R")
|
||||
|
||||
@@ -221,16 +220,10 @@ class CudaPlatformBase(Platform):
|
||||
import vllm._C_stable_libtorch # noqa: F401
|
||||
except ImportError as e:
|
||||
logger.warning_once("Failed to import from vllm._C_stable_libtorch: %r", e)
|
||||
try:
|
||||
with contextlib.suppress(ImportError):
|
||||
import vllm._moe_C_stable_libtorch # noqa: F401
|
||||
except ImportError as e:
|
||||
logger.warning_once(
|
||||
"Failed to import from vllm._moe_C_stable_libtorch: %r", e
|
||||
)
|
||||
try:
|
||||
with contextlib.suppress(ImportError):
|
||||
import vllm._qutlass_C # noqa: F401
|
||||
except ImportError as e:
|
||||
logger.warning_once("Failed to import from vllm._qutlass_C: %r", e)
|
||||
|
||||
@property
|
||||
def supported_dtypes(self) -> list[torch.dtype]:
|
||||
|
||||
Reference in New Issue
Block a user