diff --git a/csrc/cpu/torch_bindings.cpp b/csrc/cpu/torch_bindings.cpp index b93a08c01b..19068565d2 100644 --- a/csrc/cpu/torch_bindings.cpp +++ b/csrc/cpu/torch_bindings.cpp @@ -230,7 +230,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { #endif // SHM CCL -#if defined(__AVX512F__) || defined(__aarch64__) +#if defined(__AVX512F__) || (defined(__aarch64__) && !defined(__APPLE__)) ops.def("init_shm_manager(str name, int group_size, int rank) -> int", &init_shm_manager); ops.def("join_shm_manager(int handle, str name) -> str", &join_shm_manager); diff --git a/vllm/model_executor/layers/utils.py b/vllm/model_executor/layers/utils.py index 44a52b2528..364204bec5 100644 --- a/vllm/model_executor/layers/utils.py +++ b/vllm/model_executor/layers/utils.py @@ -216,8 +216,14 @@ def dispatch_cpu_unquantized_gemm( layer: torch.nn.Module, remove_weight: bool, ) -> None: + # skip for missing layers + if layer.weight.is_meta: + layer.cpu_linear = torch.nn.functional.linear + return + N, K = layer.weight.size() dtype = layer.weight.dtype + if envs.VLLM_CPU_SGL_KERNEL and check_cpu_sgl_kernel(N, K, dtype): packed_weight = torch.ops._C.convert_weight_packed(layer.weight) if getattr(layer, "bias", None) is not None: