[Core][Kernel] Share persistent workspaces for Marlin and Humming (#57421)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: ErinYin <liang.yin@daocloud.io>
This commit is contained in:
@@ -16,7 +16,6 @@ from vllm.model_executor.kernels.linear import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEExpertsModular,
|
||||
FusedMoEMethodBase,
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoeWeightScaleSupported,
|
||||
@@ -46,7 +45,6 @@ from vllm.model_executor.layers.quantization.utils.gptq_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_moe_marlin_supports_layer,
|
||||
get_marlin_input_dtype,
|
||||
marlin_make_workspace_new,
|
||||
marlin_repeat_scales_on_all_ranks,
|
||||
verify_marlin_supported,
|
||||
)
|
||||
@@ -601,12 +599,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
layer.register_parameter("w2_bias", w2_bias)
|
||||
set_weight_attrs(w2_bias, extra_weight_attrs)
|
||||
|
||||
if self.experts_cls is not None and issubclass(
|
||||
self.experts_cls, FusedMoEExpertsModular
|
||||
):
|
||||
device = layer.w13_qweight.device
|
||||
layer.workspace = marlin_make_workspace_new(device, 4)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
def replace_or_register(name: str, val: torch.Tensor | None):
|
||||
if val is None:
|
||||
|
||||
Reference in New Issue
Block a user