Load tuned fused_moe_lora shrink and expand kernel configs separately (#27435)
Signed-off-by: Yu Gong <yu3.gong@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
@@ -44,8 +44,17 @@ For `shrink`, the config file is named as `{gpu_name}_SHRINK.json`, e.g. `NVIDIA
|
||||
|
||||
For `expand`, the config fileis named as `{gpu_name}_EXPAND_{add_input}.json`, e.g. `NVIDIA_H200_EXPAND_TRUE.json`.
|
||||
|
||||
For `fused_moe_lora_w13_shrink`, the config file is named as `{gpu_name}_FUSED_MOE_LORA_W13_SHRINK.json`, e.g. `NVIDIA_H200_FUSED_MOE_LORA_W13_SHRINK.json`.
|
||||
|
||||
For `fused_moe_lora_w13_expand`, the config file is named as `{gpu_name}_FUSED_MOE_LORA_W13_EXPAND.json`, e.g. `NVIDIA_H200_FUSED_MOE_LORA_W13_EXPAND.json`.
|
||||
|
||||
For `fused_moe_lora_w2_shrink`, the config file is named as `{gpu_name}_FUSED_MOE_LORA_W2_SHRINK.json`, e.g. `NVIDIA_H200_FUSED_MOE_LORA_W2_SHRINK.json`.
|
||||
|
||||
For `fused_moe_lora_w2_expand`, the config file is named as `{gpu_name}_FUSED_MOE_LORA_W2_EXPAND.json`, e.g. `NVIDIA_H200_FUSED_MOE_LORA_W2_EXPAND.json`.
|
||||
|
||||
The `gpu_name` can be automatically detected by calling `torch.cuda.get_device_name()`
|
||||
|
||||
### Json Structure
|
||||
|
||||
Optimal kernel configuration files are saved as JSON files with the structure `config_data[max_loras][num_slices][m][k][n]`
|
||||
Optimal kernel configuration files are saved as JSON files with the structure `config_data[max_loras][num_slices][m][k][n][i]`
|
||||
where `i` is an optional dimension in the `fused_moe_lora` configuration, representing the intermediate size of the MoE layer.
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.lora.ops.triton_ops.fused_moe_lora_op import fused_moe_lora
|
||||
|
||||
from vllm.lora.ops.triton_ops.fused_moe_lora_op import (
|
||||
fused_moe_lora,
|
||||
fused_moe_lora_expand,
|
||||
fused_moe_lora_shrink,
|
||||
)
|
||||
from vllm.lora.ops.triton_ops.lora_expand_op import lora_expand
|
||||
from vllm.lora.ops.triton_ops.lora_kernel_metadata import LoRAKernelMeta
|
||||
from vllm.lora.ops.triton_ops.lora_shrink_op import lora_shrink
|
||||
@@ -11,4 +16,6 @@ __all__ = [
|
||||
"lora_shrink",
|
||||
"LoRAKernelMeta",
|
||||
"fused_moe_lora",
|
||||
"fused_moe_lora_shrink",
|
||||
"fused_moe_lora_expand",
|
||||
]
|
||||
|
||||
@@ -176,88 +176,50 @@ def _fused_moe_lora_kernel(
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def _fused_moe_lora(
|
||||
output: torch.Tensor, # (num_tokens, top_k_num, N*len(lora_a_stacked),)
|
||||
def _fused_moe_lora_shrink(
|
||||
a_intermediate_cache1: torch.Tensor,
|
||||
# (num_slices, num_tokens, top_k_num, max_lora_rank)
|
||||
qcurr_hidden_states: torch.Tensor, # (num_tokens, K,)
|
||||
lora_a_stacked: list[
|
||||
torch.Tensor
|
||||
], # [(max_loras, num_experts, max_lora_rank, K,),...]
|
||||
lora_b_stacked: list[
|
||||
torch.Tensor
|
||||
], # [(max_loras, num_experts, N, max_lora_rank,),...]
|
||||
topk_weights: torch.Tensor, # (num_tokens, top_k_num)
|
||||
sorted_token_ids: torch.Tensor, # (max_loras, _)
|
||||
expert_ids: torch.Tensor, # (max_loras, _ ,)
|
||||
num_tokens_post_padded: torch.Tensor, # (max_loras, )
|
||||
max_lora_rank: int,
|
||||
top_k_num: int,
|
||||
lora_ids: torch.Tensor,
|
||||
adapter_enabled: torch.Tensor,
|
||||
## adding for kernel
|
||||
device: torch.device,
|
||||
N: int,
|
||||
M: int,
|
||||
EM: int,
|
||||
K: int,
|
||||
num_tokens: int,
|
||||
num_experts: int,
|
||||
num_slices: int,
|
||||
block_size_m: int,
|
||||
block_size_n: int,
|
||||
block_size_k: int,
|
||||
group_size_m: int,
|
||||
num_warps: int,
|
||||
num_stages: int,
|
||||
split_k: int,
|
||||
mul_routed_weight: bool = False,
|
||||
) -> None:
|
||||
assert len(lora_a_stacked) == len(lora_b_stacked) > 0
|
||||
assert (
|
||||
sorted_token_ids.dim()
|
||||
== expert_ids.dim()
|
||||
== topk_weights.dim()
|
||||
== qcurr_hidden_states.dim()
|
||||
== 2
|
||||
)
|
||||
assert (
|
||||
sorted_token_ids.shape[0]
|
||||
== expert_ids.shape[0]
|
||||
== num_tokens_post_padded.shape[0]
|
||||
)
|
||||
assert len(lora_b_stacked) * lora_b_stacked[0].shape[-2] == output.shape[-1]
|
||||
assert output.shape[0] == topk_weights.shape[0]
|
||||
assert top_k_num == topk_weights.shape[1]
|
||||
w1_lora_a_stacked = lora_a_stacked[0]
|
||||
|
||||
for lora_a, lora_b in zip(lora_a_stacked, lora_b_stacked):
|
||||
assert lora_a.dtype == lora_b.dtype == output.dtype == qcurr_hidden_states.dtype
|
||||
assert lora_a.dtype in [torch.float16, torch.bfloat16]
|
||||
|
||||
device = qcurr_hidden_states.device
|
||||
num_slices = len(lora_a_stacked)
|
||||
|
||||
config = {
|
||||
shrink_config = {
|
||||
"BLOCK_SIZE_M": block_size_m,
|
||||
"BLOCK_SIZE_N": block_size_n,
|
||||
"BLOCK_SIZE_K": block_size_k,
|
||||
"GROUP_SIZE_M": group_size_m,
|
||||
"num_warps": num_warps,
|
||||
"num_stages": num_stages,
|
||||
"SPLIT_K": split_k,
|
||||
}
|
||||
|
||||
w1_lora_a_stacked = lora_a_stacked[0]
|
||||
w1_lora_b_stacked = lora_b_stacked[0]
|
||||
num_experts = lora_a_stacked[0].shape[1]
|
||||
|
||||
N = max_lora_rank
|
||||
M = topk_weights.shape[0]
|
||||
EM = sorted_token_ids.shape[1]
|
||||
K = qcurr_hidden_states.shape[1]
|
||||
num_tokens = M * top_k_num
|
||||
w1_output_dim_size = w1_lora_b_stacked.shape[2]
|
||||
|
||||
lora_intermediate_cache1 = torch.zeros(
|
||||
(num_slices * M * top_k_num * (max_lora_rank + w1_output_dim_size)),
|
||||
dtype=output.dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
# slices
|
||||
a_intermediate_size = num_slices * M * top_k_num * max_lora_rank
|
||||
a_intermediate_cache1 = lora_intermediate_cache1[:a_intermediate_size].view(
|
||||
num_slices, M, top_k_num, max_lora_rank
|
||||
)
|
||||
b_intermediate_cache1 = lora_intermediate_cache1[a_intermediate_size:].view(
|
||||
num_slices, M, top_k_num, w1_output_dim_size
|
||||
)
|
||||
|
||||
b_ptr = _get_ptr(lora_a_stacked, device)
|
||||
|
||||
grid = lambda META: (
|
||||
@@ -299,19 +261,70 @@ def _fused_moe_lora(
|
||||
num_slice_c=num_slices,
|
||||
top_k=1 if mul_routed_weight else top_k_num,
|
||||
MUL_ROUTED_WEIGHT=False,
|
||||
**config,
|
||||
**shrink_config,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def _fused_moe_lora_expand(
|
||||
output: torch.Tensor, # (num_tokens, top_k_num, N*len(lora_a_stacked),)
|
||||
a_intermediate_cache1: torch.Tensor, # (num_slices, M, top_k_num, max_lora_rank)
|
||||
lora_b_stacked: list[
|
||||
torch.Tensor
|
||||
], # [(max_loras, num_experts, max_lora_rank, K,),...]
|
||||
topk_weights: torch.Tensor, # (num_tokens, top_k_num)
|
||||
sorted_token_ids: torch.Tensor, # (max_loras, _)
|
||||
expert_ids: torch.Tensor, # (max_loras, _ ,)
|
||||
num_tokens_post_padded: torch.Tensor, # (max_loras, )
|
||||
top_k_num: int,
|
||||
lora_ids: torch.Tensor,
|
||||
adapter_enabled: torch.Tensor,
|
||||
## adding for kernel
|
||||
device: torch.device,
|
||||
N: int,
|
||||
M: int,
|
||||
EM: int,
|
||||
K: int,
|
||||
num_tokens: int,
|
||||
num_experts: int,
|
||||
num_slices: int,
|
||||
max_lora_rank: int,
|
||||
w1_output_dim_size: int,
|
||||
block_size_m: int,
|
||||
block_size_n: int,
|
||||
block_size_k: int,
|
||||
group_size_m: int,
|
||||
num_warps: int,
|
||||
num_stages: int,
|
||||
split_k: int,
|
||||
mul_routed_weight: bool = False,
|
||||
) -> None:
|
||||
b_ptr = _get_ptr(lora_b_stacked, device)
|
||||
K = max_lora_rank
|
||||
N = w1_output_dim_size
|
||||
|
||||
w1_lora_b_stacked = lora_b_stacked[0]
|
||||
|
||||
a_intermediate_cache1 = a_intermediate_cache1.view(
|
||||
-1, a_intermediate_cache1.shape[3]
|
||||
)
|
||||
|
||||
# Set split_k = 1 for expand calls
|
||||
config["SPLIT_K"] = 1
|
||||
b_intermediate_cache1 = torch.zeros(
|
||||
(num_slices, M, top_k_num, w1_output_dim_size),
|
||||
dtype=output.dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
expand_config = {
|
||||
"BLOCK_SIZE_M": block_size_m,
|
||||
"BLOCK_SIZE_N": block_size_n,
|
||||
"BLOCK_SIZE_K": block_size_k,
|
||||
"GROUP_SIZE_M": group_size_m,
|
||||
"num_warps": num_warps,
|
||||
"num_stages": num_stages,
|
||||
"SPLIT_K": split_k, # Set split_k = 1 for expand calls
|
||||
}
|
||||
|
||||
grid = lambda META: (
|
||||
triton.cdiv(EM, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]),
|
||||
len(lora_b_stacked),
|
||||
@@ -348,12 +361,142 @@ def _fused_moe_lora(
|
||||
num_slice_c=num_slices,
|
||||
top_k=1,
|
||||
MUL_ROUTED_WEIGHT=mul_routed_weight,
|
||||
**config,
|
||||
**expand_config,
|
||||
)
|
||||
for i in range(num_slices):
|
||||
output[:, :, i * N : (i + 1) * N] += b_intermediate_cache1[i]
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def _fused_moe_lora(
|
||||
output: torch.Tensor, # (num_tokens, top_k_num, N*len(lora_a_stacked),)
|
||||
qcurr_hidden_states: torch.Tensor, # (num_tokens, K,)
|
||||
lora_a_stacked: list[
|
||||
torch.Tensor
|
||||
], # [(max_loras, num_experts, max_lora_rank, K,),...]
|
||||
lora_b_stacked: list[
|
||||
torch.Tensor
|
||||
], # [(max_loras, num_experts, N, max_lora_rank,),...]
|
||||
topk_weights: torch.Tensor, # (num_tokens, top_k_num)
|
||||
sorted_token_ids: torch.Tensor, # (max_loras, _)
|
||||
expert_ids: torch.Tensor, # (max_loras, _ ,)
|
||||
num_tokens_post_padded: torch.Tensor, # (max_loras, )
|
||||
max_lora_rank: int,
|
||||
top_k_num: int,
|
||||
lora_ids: torch.Tensor,
|
||||
adapter_enabled: torch.Tensor,
|
||||
shrink_block_size_m: int,
|
||||
shrink_block_size_n: int,
|
||||
shrink_block_size_k: int,
|
||||
shrink_group_size_m: int,
|
||||
shrink_num_warps: int,
|
||||
shrink_num_stages: int,
|
||||
shrink_split_k: int,
|
||||
expand_block_size_m: int,
|
||||
expand_block_size_n: int,
|
||||
expand_block_size_k: int,
|
||||
expand_group_size_m: int,
|
||||
expand_num_warps: int,
|
||||
expand_num_stages: int,
|
||||
expand_split_k: int,
|
||||
mul_routed_weight: bool = False,
|
||||
) -> None:
|
||||
assert len(lora_a_stacked) == len(lora_b_stacked) > 0
|
||||
assert (
|
||||
sorted_token_ids.dim()
|
||||
== expert_ids.dim()
|
||||
== topk_weights.dim()
|
||||
== qcurr_hidden_states.dim()
|
||||
== 2
|
||||
)
|
||||
assert (
|
||||
sorted_token_ids.shape[0]
|
||||
== expert_ids.shape[0]
|
||||
== num_tokens_post_padded.shape[0]
|
||||
)
|
||||
assert len(lora_b_stacked) * lora_b_stacked[0].shape[-2] == output.shape[-1]
|
||||
assert output.shape[0] == topk_weights.shape[0]
|
||||
assert top_k_num == topk_weights.shape[1]
|
||||
device = qcurr_hidden_states.device
|
||||
num_slices = len(lora_a_stacked)
|
||||
w1_lora_b_stacked = lora_b_stacked[0]
|
||||
num_experts = lora_a_stacked[0].shape[1]
|
||||
N = max_lora_rank
|
||||
M = topk_weights.shape[0]
|
||||
EM = sorted_token_ids.shape[1]
|
||||
K = qcurr_hidden_states.shape[1]
|
||||
num_tokens = M * top_k_num
|
||||
w1_output_dim_size = w1_lora_b_stacked.shape[2]
|
||||
|
||||
a_intermediate_cache1 = torch.zeros(
|
||||
(num_slices, M, top_k_num, max_lora_rank),
|
||||
dtype=output.dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
_fused_moe_lora_shrink(
|
||||
a_intermediate_cache1,
|
||||
qcurr_hidden_states,
|
||||
lora_a_stacked,
|
||||
topk_weights,
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_padded,
|
||||
top_k_num,
|
||||
lora_ids,
|
||||
adapter_enabled,
|
||||
## adding for kernel
|
||||
device,
|
||||
N,
|
||||
M,
|
||||
EM,
|
||||
K,
|
||||
num_tokens,
|
||||
num_experts,
|
||||
num_slices,
|
||||
shrink_block_size_m,
|
||||
shrink_block_size_n,
|
||||
shrink_block_size_k,
|
||||
shrink_group_size_m,
|
||||
shrink_num_warps,
|
||||
shrink_num_stages,
|
||||
shrink_split_k,
|
||||
mul_routed_weight,
|
||||
)
|
||||
|
||||
_fused_moe_lora_expand(
|
||||
output,
|
||||
a_intermediate_cache1,
|
||||
lora_b_stacked,
|
||||
topk_weights,
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_padded,
|
||||
top_k_num,
|
||||
lora_ids,
|
||||
adapter_enabled,
|
||||
## adding for kernel
|
||||
device,
|
||||
N,
|
||||
M,
|
||||
EM,
|
||||
K,
|
||||
num_tokens,
|
||||
num_experts,
|
||||
num_slices,
|
||||
max_lora_rank,
|
||||
w1_output_dim_size,
|
||||
expand_block_size_m,
|
||||
expand_block_size_n,
|
||||
expand_block_size_k,
|
||||
expand_group_size_m,
|
||||
expand_num_warps,
|
||||
expand_num_stages,
|
||||
expand_split_k,
|
||||
mul_routed_weight,
|
||||
)
|
||||
|
||||
|
||||
def _fused_moe_lora_fake(
|
||||
output: torch.Tensor,
|
||||
qcurr_hidden_states: torch.Tensor,
|
||||
@@ -367,10 +510,84 @@ def _fused_moe_lora_fake(
|
||||
top_k_num: int,
|
||||
lora_ids: torch.Tensor,
|
||||
adapter_enabled: torch.Tensor,
|
||||
shrink_block_size_m: int,
|
||||
shrink_block_size_n: int,
|
||||
shrink_block_size_k: int,
|
||||
shrink_group_size_m: int,
|
||||
shrink_num_warps: int,
|
||||
shrink_num_stages: int,
|
||||
shrink_split_k: int,
|
||||
expand_block_size_m: int,
|
||||
expand_block_size_n: int,
|
||||
expand_block_size_k: int,
|
||||
expand_group_size_m: int,
|
||||
expand_num_warps: int,
|
||||
expand_num_stages: int,
|
||||
expand_split_k: int,
|
||||
mul_routed_weight: bool = False,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
def _fused_moe_lora_shrink_fake(
|
||||
a_intermediate_cache1: torch.Tensor,
|
||||
qcurr_hidden_states: torch.Tensor,
|
||||
lora_a_stacked: list[torch.Tensor],
|
||||
topk_weights: torch.Tensor,
|
||||
sorted_token_ids: torch.Tensor,
|
||||
expert_ids: torch.Tensor,
|
||||
num_tokens_post_padded: torch.Tensor,
|
||||
top_k_num: int,
|
||||
lora_ids: torch.Tensor,
|
||||
adapter_enabled: torch.Tensor,
|
||||
device: torch.device,
|
||||
N: int,
|
||||
M: int,
|
||||
EM: int,
|
||||
K: int,
|
||||
num_tokens: int,
|
||||
num_experts: int,
|
||||
num_slices: int,
|
||||
block_size_m: int,
|
||||
block_size_n: int,
|
||||
block_size_k: int,
|
||||
group_size_m: int,
|
||||
num_warps: int,
|
||||
num_stages: int,
|
||||
split_k: int,
|
||||
mul_routed_weight: bool = False,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
def _fused_moe_lora_expand_fake(
|
||||
output: torch.Tensor,
|
||||
a_intermediate_cache1: torch.Tensor,
|
||||
lora_b_stacked: list[torch.Tensor],
|
||||
topk_weights: torch.Tensor,
|
||||
sorted_token_ids: torch.Tensor,
|
||||
expert_ids: torch.Tensor,
|
||||
num_tokens_post_padded: torch.Tensor,
|
||||
top_k_num: int,
|
||||
lora_ids: torch.Tensor,
|
||||
adapter_enabled: torch.Tensor,
|
||||
device: torch.device,
|
||||
N: int,
|
||||
M: int,
|
||||
EM: int,
|
||||
K: int,
|
||||
num_tokens: int,
|
||||
num_experts: int,
|
||||
num_slices: int,
|
||||
max_lora_rank: int,
|
||||
w1_output_dim_size: int,
|
||||
block_size_m: int,
|
||||
block_size_n: int,
|
||||
block_size_k: int,
|
||||
group_size_m: int,
|
||||
num_warps: int,
|
||||
num_stages: int,
|
||||
split_k: int,
|
||||
mul_routed_weight: bool = False,
|
||||
) -> None:
|
||||
return
|
||||
@@ -383,7 +600,26 @@ try:
|
||||
mutates_args=["output"],
|
||||
fake_impl=_fused_moe_lora_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_moe_lora_shrink",
|
||||
op_func=_fused_moe_lora_shrink,
|
||||
mutates_args=["a_intermediate_cache1"],
|
||||
fake_impl=_fused_moe_lora_shrink_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_moe_lora_expand",
|
||||
op_func=_fused_moe_lora_expand,
|
||||
mutates_args=["output"],
|
||||
fake_impl=_fused_moe_lora_expand_fake,
|
||||
)
|
||||
|
||||
fused_moe_lora = torch.ops.vllm.fused_moe_lora
|
||||
fused_moe_lora_shrink = torch.ops.vllm.fused_moe_lora_shrink
|
||||
fused_moe_lora_expand = torch.ops.vllm.fused_moe_lora_expand
|
||||
|
||||
except AttributeError:
|
||||
fused_moe_lora = _fused_moe_lora
|
||||
fused_moe_lora_shrink = _fused_moe_lora_shrink
|
||||
fused_moe_lora_expand = _fused_moe_lora_expand
|
||||
|
||||
@@ -154,13 +154,13 @@ def load_lora_op_config(op_type: str, add_inputs: bool | None) -> dict | None:
|
||||
gpu_name = gpu_name.replace("-", "_")
|
||||
|
||||
config_fname = None
|
||||
if op_type == "shrink":
|
||||
config_fname = f"{gpu_name}_{op_type.upper()}.json"
|
||||
else:
|
||||
assert op_type == "expand"
|
||||
# only expand op needs to consider add_inputs
|
||||
if op_type == "expand":
|
||||
config_fname = (
|
||||
f"{gpu_name}_{op_type.upper()}_{str(add_inputs).upper()}.json"
|
||||
)
|
||||
else:
|
||||
config_fname = f"{gpu_name}_{op_type.upper()}.json"
|
||||
|
||||
config_path = Path(f"{user_defined_config_folder}/{config_fname}")
|
||||
if not config_path.exists():
|
||||
@@ -186,8 +186,17 @@ def get_lora_op_configs(
|
||||
rank: int,
|
||||
num_slices: int,
|
||||
add_inputs: bool | None = None,
|
||||
moe_intermediate_size: int | None = None,
|
||||
) -> dict[str, int | None]:
|
||||
assert op_type in ["shrink", "expand"]
|
||||
# Add support for fused_moe_lora ops
|
||||
assert op_type in [
|
||||
"shrink",
|
||||
"expand",
|
||||
"fused_moe_lora_w13_shrink",
|
||||
"fused_moe_lora_w13_expand",
|
||||
"fused_moe_lora_w2_shrink",
|
||||
"fused_moe_lora_w2_expand",
|
||||
]
|
||||
|
||||
# default config
|
||||
default = {}
|
||||
@@ -203,6 +212,22 @@ def get_lora_op_configs(
|
||||
"num_stages": 2,
|
||||
"max_nreg": None,
|
||||
}
|
||||
# The default config for fused_moe_lora ops
|
||||
elif op_type in [
|
||||
"fused_moe_lora_w13_shrink",
|
||||
"fused_moe_lora_w13_expand",
|
||||
"fused_moe_lora_w2_shrink",
|
||||
"fused_moe_lora_w2_expand",
|
||||
]:
|
||||
default = {
|
||||
"block_m": 64,
|
||||
"block_n": 64,
|
||||
"block_k": 32,
|
||||
"num_warps": 4,
|
||||
"num_stages": 3,
|
||||
"group_size_m": 8,
|
||||
"split_k": 1,
|
||||
}
|
||||
else:
|
||||
default = {
|
||||
"block_m": 64,
|
||||
@@ -247,5 +272,13 @@ def get_lora_op_configs(
|
||||
or config_data[min(config_data.keys(), key=lambda x: abs(int(x) - n))]
|
||||
)
|
||||
|
||||
# slice by moe-intermediate-size if applicable
|
||||
if moe_intermediate_size is not None:
|
||||
i = moe_intermediate_size
|
||||
config_data = (
|
||||
config_data.get(str(i))
|
||||
or config_data[min(config_data.keys(), key=lambda x: abs(int(x) - i))]
|
||||
)
|
||||
|
||||
assert config_data is not None
|
||||
return config_data
|
||||
|
||||
Reference in New Issue
Block a user