Line data    Source code

       1             : !    This file is part of ELPA.
       2             : !
       3             : !    The ELPA library was originally created by the ELPA consortium,
       4             : !    consisting of the following organizations:
       5             : !
       6             : !    - Rechenzentrum Garching der Max-Planck-Gesellschaft (RZG),
       7             : !    - Bergische Universität Wuppertal, Lehrstuhl für angewandte
       8             : !      Informatik,
       9             : !    - Technische Universität München, Lehrstuhl für Informatik mit
      10             : !      Schwerpunkt Wissenschaftliches Rechnen ,
      11             : !    - Fritz-Haber-Institut, Berlin, Abt. Theorie,
      12             : !    - Max-Plack-Institut für Mathematik in den Naturwissenschaften,
      13             : !      Leipzig, Abt. Komplexe Strukutren in Biologie und Kognition,
      14             : !      and
      15             : !    - IBM Deutschland GmbH
      16             : !
      17             : !
      18             : !    More information can be found here:
      19             : !    http://elpa.rzg.mpg.de/
      20             : !
      21             : !    ELPA is free software: you can redistribute it and/or modify
      22             : !    it under the terms of the version 3 of the license of the
      23             : !    GNU Lesser General Public License as published by the Free
      24             : !    Software Foundation.
      25             : !
      26             : !    ELPA is distributed in the hope that it will be useful,
      27             : !    but WITHOUT ANY WARRANTY; without even the implied warranty of
      28             : !    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      29             : !    GNU Lesser General Public License for more details.
      30             : !
      31             : !    You should have received a copy of the GNU Lesser General Public License
      32             : !    along with ELPA.  If not, see <http://www.gnu.org/licenses/>
      33             : !
      34             : !    ELPA reflects a substantial effort on the part of the original
      35             : !    ELPA consortium, and we ask you to respect the spirit of the
      36             : !    license that we chose: i.e., please contribute any changes you
      37             : !    may have back to the original ELPA library distribution, and keep
      38             : !    any derivatives of ELPA under the same license that we chose for
      39             : !    the original distribution, the GNU Lesser General Public License.
      40             : !
      41             : ! This file was written by A. Marek, MPCDF
      42             : 
      43             : #include "config-f90.h"
      44             : 
      45             : module mod_check_for_gpu
      46             : 
      47             :   contains
      48             : 
      49           0 :     function check_for_gpu(myid, numberOfDevices, wantDebug) result(gpuAvailable)
      50             :       use cuda_functions
      51             :       use precision
      52             :       use elpa_mpi
      53             :       implicit none
      54             : 
      55             :       integer(kind=ik), intent(in)  :: myid
      56             :       logical, optional, intent(in) :: wantDebug
      57             :       logical                       :: success, wantDebugMessage
      58             :       integer(kind=ik), intent(out) :: numberOfDevices
      59             :       integer(kind=ik)              :: deviceNumber, mpierr, maxNumberOfDevices
      60             :       logical                       :: gpuAvailable
      61             :       !character(len=1024)           :: envname
      62             : 
      63           0 :       gpuAvailable = .false.
      64             : 
      65           0 :       if (.not.(present(wantDebug))) then
      66           0 :         wantDebugMessage = .false.
      67             :       else
      68           0 :         if (wantDebug) then
      69           0 :           wantDebugMessage=.true.
      70             :         else
      71           0 :           wantDebugMessage=.false.
      72             :         endif
      73             :       endif
      74             : 
      75             :       ! call getenv("CUDA_PROXY_PIPE_DIRECTORY", envname)
      76           0 :       success = cuda_getdevicecount(numberOfDevices)
      77             : 
      78           0 :       if (.not.(success)) then
      79           0 :         print *,"error in cuda_getdevicecount"
      80           0 :         stop 1
      81             :       endif
      82             : 
      83             :       ! make sure that all nodes have the same number of GPU's, otherwise
      84             :       ! we run into loadbalancing trouble
      85             : #ifdef WITH_MPI
      86           0 :       call mpi_allreduce(numberOfDevices, maxNumberOfDevices, 1, MPI_INTEGER, MPI_MAX, MPI_COMM_WORLD, mpierr)
      87             : 
      88           0 :       if (maxNumberOfDevices .ne. numberOfDevices) then
      89           0 :         print *,"Different number of GPU devices on MPI tasks!"
      90           0 :         print *,"GPUs will NOT be used!"
      91           0 :         gpuAvailable = .false.
      92           0 :         return
      93             :       endif
      94             : #endif
      95           0 :       if (numberOfDevices .ne. 0) then
      96           0 :         gpuAvailable = .true.
      97             :         ! Usage of GPU is possible since devices have been detected
      98             : 
      99           0 :         if (myid==0) then
     100           0 :           if (wantDebugMessage) then
     101           0 :             print *
     102           0 :             print '(3(a,i0))','Found ', numberOfDevices, ' GPUs'
     103             :           endif
     104             :         endif
     105             : 
     106           0 :         deviceNumber = mod(myid, numberOfDevices)
     107           0 :         success = cuda_setdevice(deviceNumber)
     108             : 
     109           0 :         if (.not.(success)) then
     110           0 :           print *,"Cannot set CudaDevice"
     111           0 :           stop 1
     112             :         endif
     113           0 :         if (wantDebugMessage) then
     114           0 :           print '(3(a,i0))', 'MPI rank ', myid, ' uses GPU #', deviceNumber
     115             :         endif
     116             :         
     117           0 :         success = cublas_create(cublasHandle)
     118           0 :         if (.not.(success)) then
     119           0 :           print *,"Cannot create cublas handle"
     120           0 :           stop 1
     121             :         endif
     122             :         
     123             :       endif
     124             : 
     125           0 :     end function
     126             : end module