[Core][Kernel] Share persistent workspaces for Marlin and Humming (#57421)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: ErinYin <liang.yin@daocloud.io>
This commit is contained in:
Misha Goin
2026-09-20 17:28:33 +08:00
committed by GitHub
co-authored by ErinYin
parent 0e110f696d
commit e5fce7b56b
21 changed files with 598 additions and 211 deletions
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
from vllm.model_executor.layers.fused_moe.moe_fused_mul_sum import moe_fused_mul_sum
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
MoEPermuteScratch,
get_moe_permute_scratch,
moe_permute,
moe_permute_unpermute_supported,
moe_unpermute,
@@ -176,7 +177,6 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
max_num_tokens=max_num_tokens,
num_dispatchers=num_dispatchers,
)
self._permute_scratch: dict[int, MoEPermuteScratch] = {}
def init_humming_moe(self):
from vllm.utils.humming import get_heuristics_config
@@ -259,24 +259,20 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
if not moe_permute_unpermute_supported():
return None
scratch = self._permute_scratch.get(topk)
if scratch is None:
max_expanded_rows = (
self.moe_config.max_num_tokens
* self.moe_config.dp_size
* self.moe_config.experts_per_token
)
scratch = MoEPermuteScratch(
max_num_tokens=math.ceil(max_expanded_rows / topk),
topk=topk,
num_experts=self.moe_config.num_experts,
num_local_experts=self.moe_config.num_local_experts,
device=torch.device(self.moe_config.device),
hidden_size=self.moe_config.hidden_dim,
hidden_dtype=self.moe_config.in_dtype,
)
self._permute_scratch[topk] = scratch
return scratch
max_expanded_rows = (
self.moe_config.max_num_tokens
* self.moe_config.dp_size
* self.moe_config.experts_per_token
)
return get_moe_permute_scratch(
max_num_tokens=math.ceil(max_expanded_rows / topk),
topk=topk,
num_experts=self.moe_config.num_experts,
num_local_experts=self.moe_config.num_local_experts,
device=torch.device(self.moe_config.device),
hidden_size=self.moe_config.hidden_dim,
hidden_dtype=self.moe_config.in_dtype,
)
def get_global_valid_shape_m(self, topk_ids: torch.Tensor):
ctx = get_forward_context()
@@ -35,7 +35,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
from vllm.model_executor.layers.fused_moe.utils import _resize_cache
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
get_marlin_input_dtype,
marlin_make_workspace_new,
get_marlin_workspace,
marlin_moe_intermediate_size,
marlin_quant_input,
)
@@ -95,7 +95,7 @@ def _fused_marlin_moe(
N = marlin_moe_intermediate_size(w1, w2)
w13_num_shards = 2 if activation.is_gated else 1
if workspace is None:
workspace = marlin_make_workspace_new(hidden_states.device, 4)
workspace = get_marlin_workspace(hidden_states.device)
if intermediate_cache13 is None:
intermediate_cache13 = torch.empty(