diff --git a/vllm/model_executor/layers/fused_moe/moe_permute_unpermute.py b/vllm/model_executor/layers/fused_moe/moe_permute_unpermute.py index df43068943..ad9fb509a1 100644 --- a/vllm/model_executor/layers/fused_moe/moe_permute_unpermute.py +++ b/vllm/model_executor/layers/fused_moe/moe_permute_unpermute.py @@ -74,6 +74,9 @@ class MoEPermuteScratch: self.sort_workspace = torch.empty( sorter_size, dtype=torch.int8, device=self.device ) + # torch.device("cuda") in config, after initialized, + # will be changed to cuda:{index}, so we need to refresh here. + self.device = self.token_expert_indices.device def validate(self, hidden_states: torch.Tensor, topk_ids: torch.Tensor) -> None: n_token, n_hidden = hidden_states.shape