[Bugfix] Fix Marlin MoE act order when is_k_full == False (#8741)
Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
This commit is contained in:
co-authored by
Tyler Michael Smith
parent
5bf8789b2a
commit
d081da0064
@@ -0,0 +1,3 @@
|
||||
#pragma once
|
||||
|
||||
#define VLLM_IMPLIES(p, q) (!(p) || (q))
|
||||
@@ -25,6 +25,7 @@
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "core/exception.hpp"
|
||||
#include "core/scalar_type.hpp"
|
||||
#include "marlin_kernels/marlin_moe_kernel_ku4b8.h"
|
||||
#include "marlin_kernels/marlin_moe_kernel_ku8b128.h"
|
||||
@@ -189,7 +190,7 @@ int get_scales_cache_size(thread_config_t const& th_config, int prob_m,
|
||||
int load_groups =
|
||||
tb_groups * STAGES * 2; // Chunk size is 2x pipeline over dim K
|
||||
load_groups = max(load_groups, 32); // We load at least 32 scale groups
|
||||
return load_groups * tb_n * 2;
|
||||
return load_groups * tb_n * 4;
|
||||
|
||||
} else {
|
||||
int tb_scales = tb_groups * tb_n * 2;
|
||||
@@ -433,11 +434,7 @@ void marlin_mm_moe(const void* A, const void* B, void* C,
|
||||
int4* C_ptr = (int4*)C;
|
||||
const float* topk_weights_ptr = (const float*)topk_weights;
|
||||
const int* sorted_ids_ptr = (const int*)sorted_ids;
|
||||
const int4* s_ptr =
|
||||
(const int4*)s +
|
||||
(((group_size == -1 || group_size == 0) ? 1 : prob_k / group_size) *
|
||||
prob_n / 8) *
|
||||
expert_idx;
|
||||
const int4* s_ptr = (const int4*)s + num_groups * prob_n / 8 * expert_idx;
|
||||
const int* g_idx_ptr = (const int*)g_idx + prob_k * expert_idx;
|
||||
const int* perm_ptr = (const int*)perm + prob_k * expert_idx;
|
||||
int* locks = (int*)workspace;
|
||||
@@ -521,6 +518,9 @@ torch::Tensor marlin_gemm_moe(
|
||||
" is not size_n = ", size_n);
|
||||
num_groups = b_scales.size(1);
|
||||
|
||||
TORCH_CHECK(VLLM_IMPLIES(!is_k_full, has_act_order),
|
||||
"if is_k_full is false, has_act_order must be true");
|
||||
|
||||
if (has_act_order) {
|
||||
if (is_k_full) {
|
||||
TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1");
|
||||
|
||||
Reference in New Issue
Block a user