diff --git a/cmake/utils.cmake b/cmake/utils.cmake
index 6ded936213..83eed340eb 100644
--- a/cmake/utils.cmake
+++ b/cmake/utils.cmake
@@ -425,6 +425,7 @@ function(cuda_archs_loose_intersection OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_AR
       if("${_base}" IN_LIST _TGT_CUDA_ARCHS)
         list(REMOVE_ITEM _SRC_CUDA_ARCHS "${_arch}")
         list(REMOVE_ITEM _TGT_CUDA_ARCHS "${_base}")
+        list(REMOVE_ITEM _TGT_CUDA_ARCHS "${_base}a" "${_base}f")
         list(APPEND _CUDA_ARCHS "${_arch}")
       endif()
     endif()
@@ -464,8 +465,10 @@ function(cuda_archs_loose_intersection OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_AR
       string(REGEX REPLACE "[af]$" "" _base "${_arch}")
       if ("${_base}" IN_LIST _SRC_CUDA_ARCHS)
         list(REMOVE_ITEM _TGT_CUDA_ARCHS "${_arch}")
-        list(REMOVE_ITEM _SRC_CUDA_ARCHS "${_base}")
         list(APPEND _CUDA_ARCHS "${_arch}")
+        if (NOT "${_base}" IN_LIST _TGT_CUDA_ARCHS)
+          list(REMOVE_ITEM _SRC_CUDA_ARCHS "${_base}")
+        endif()
       endif()
     endif()
   endforeach()