[Core][Kernel] Share persistent workspaces for Marlin and Humming (#57421)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: ErinYin <liang.yin@daocloud.io>
This commit is contained in:
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
|
||||
from vllm.model_executor.layers.fused_moe.moe_fused_mul_sum import moe_fused_mul_sum
|
||||
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
|
||||
MoEPermuteScratch,
|
||||
get_moe_permute_scratch,
|
||||
moe_permute,
|
||||
moe_permute_unpermute_supported,
|
||||
moe_unpermute,
|
||||
@@ -176,7 +177,6 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=num_dispatchers,
|
||||
)
|
||||
self._permute_scratch: dict[int, MoEPermuteScratch] = {}
|
||||
|
||||
def init_humming_moe(self):
|
||||
from vllm.utils.humming import get_heuristics_config
|
||||
@@ -259,24 +259,20 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
if not moe_permute_unpermute_supported():
|
||||
return None
|
||||
|
||||
scratch = self._permute_scratch.get(topk)
|
||||
if scratch is None:
|
||||
max_expanded_rows = (
|
||||
self.moe_config.max_num_tokens
|
||||
* self.moe_config.dp_size
|
||||
* self.moe_config.experts_per_token
|
||||
)
|
||||
scratch = MoEPermuteScratch(
|
||||
max_num_tokens=math.ceil(max_expanded_rows / topk),
|
||||
topk=topk,
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
hidden_size=self.moe_config.hidden_dim,
|
||||
hidden_dtype=self.moe_config.in_dtype,
|
||||
)
|
||||
self._permute_scratch[topk] = scratch
|
||||
return scratch
|
||||
max_expanded_rows = (
|
||||
self.moe_config.max_num_tokens
|
||||
* self.moe_config.dp_size
|
||||
* self.moe_config.experts_per_token
|
||||
)
|
||||
return get_moe_permute_scratch(
|
||||
max_num_tokens=math.ceil(max_expanded_rows / topk),
|
||||
topk=topk,
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
hidden_size=self.moe_config.hidden_dim,
|
||||
hidden_dtype=self.moe_config.in_dtype,
|
||||
)
|
||||
|
||||
def get_global_valid_shape_m(self, topk_ids: torch.Tensor):
|
||||
ctx = get_forward_context()
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
from vllm.model_executor.layers.fused_moe.utils import _resize_cache
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
get_marlin_input_dtype,
|
||||
marlin_make_workspace_new,
|
||||
get_marlin_workspace,
|
||||
marlin_moe_intermediate_size,
|
||||
marlin_quant_input,
|
||||
)
|
||||
@@ -95,7 +95,7 @@ def _fused_marlin_moe(
|
||||
N = marlin_moe_intermediate_size(w1, w2)
|
||||
w13_num_shards = 2 if activation.is_gated else 1
|
||||
if workspace is None:
|
||||
workspace = marlin_make_workspace_new(hidden_states.device, 4)
|
||||
workspace = get_marlin_workspace(hidden_states.device)
|
||||
|
||||
if intermediate_cache13 is None:
|
||||
intermediate_cache13 = torch.empty(
|
||||
|
||||
Reference in New Issue
Block a user