Changes to make the PGI compiler work with FLEUR. Unfortunately

eig-file/data IO is broken and hence the version is not useful yet.

cdn/cdnval.F90

@@ -528,12 +528,14 @@ CONTAINS
                 zMat%nbands = dimension%neigd
              END IF
              zMat%z_r = 0
+#ifndef __PGI
              CALL cdn_read(&
                eig_id,dimension%nvd,dimension%jspd,mpi%irank,mpi%isize,&
                ikpt,jspin,dimension%nbasfcn,noco%l_ss,noco%l_noco,&
                noccbd,n_start,n_end,&
                lapw%nmat,lapw%nv,ello,evdu,epar,kveclo,&
                lapw%k1,lapw%k2,lapw%k3,bkpt,wk,nbands,eig,zMat%z_r)
+#endif
           ELSE
              IF (.NOT.ALLOCATED(zMat%z_c)) THEN
                 ALLOCATE (zMat%z_c(dimension%nbasfcn,dimension%neigd))
@@ -541,12 +543,14 @@ CONTAINS
                 zMat%nbands = dimension%neigd
              END IF
              zMat%z_c = 0
+#ifndef __PGI
              CALL cdn_read(&
                eig_id,dimension%nvd,dimension%jspd,mpi%irank,mpi%isize,&
                ikpt,jspin,dimension%nbasfcn,noco%l_ss,noco%l_noco,&
                noccbd,n_start,n_end,&
                lapw%nmat,lapw%nv,ello,evdu,epar,kveclo,&
                lapw%k1,lapw%k2,lapw%k3,bkpt,wk,nbands,eig,zMat%z_c)
+#endif
           endif
           !IF (l_evp.AND.(isize.GT.1)) THEN
           !  eig(1:noccbd) = eig(n_start:n_end)

cmake/CompilerConfig.txt

-set(FLEUR_DEFINITIONS "CPP_DOUBLE")
-set(FLEUR_MPI_DEFINITIONS "CPP_DOUBLE")
+set(FLEUR_DEFINITIONS ${FLEUR_DEFINITIONS} "CPP_DOUBLE")
+set(FLEUR_MPI_DEFINITIONS ${FLEUR_MPI_DEFINITIONS} "CPP_DOUBLE")
 
 include("cmake/compilerflags.cmake")
 include("cmake/tests/test_XML.cmake")

cmake/JUQEEN.cmake

@@ -4,4 +4,7 @@ set(CMAKE_C_COMPILER mpixlc)
 #Add include pathes
 set(FLEUR_Fortran_FLAGS "-I$ENV{ELPA_INCLUDE} -I$ENV{HDF5_DIR}/include")
 #Add linker stuff
-set(FLEUR_LIBRARIES "-L$ENV{SCALAPACK_ROOT}/lib;-lelpa;-lscalapack;-L/bgsys/local/lapack/3.3.0_g/lib;-llapack;-L/bgsys/local/lib;-qessl;-lesslsmpbg;-L$ENV{XML2LIB};-lxml2;-L$ENV{HDF5_DIR}/lib;-lhdf5_fortran;-lhdf5;-L/bgsys/local/zlib/lib/;-lz;-L/bgsys/local/szip/lib/;-lsz")
+set(FLEUR_LIBRARIES "-L$ENV{SCALAPACK_ROOT}/lib;-lelpa;-lscalapack;-L/bgsys/local/lapack/3.3.0_g/lib;-llapack;-L/bgsys/local/lib;-qessl;-lesslsmpbg;-L$ENV{XML2LIB};-lxml2;-L$ENV{HDF5_DIR}/lib;-lhdf5_fortran;-lhdf5;-L/bgsys/local/zlib/lib/;-lz;-L/bgsys/local/szip/lib/;-lsz")
\ No newline at end of file
+set(FLEUR_DEFINITIONS  "CPP_AIX" )
+set(FLEUR_MPI_DEFINITIONS  "CPP_AIX" )
+

cmake/JURECA.cmake

@@ -2,6 +2,8 @@
 set(CMAKE_Fortran_COMPILER mpif90)
 set(CMAKE_C_COMPILER mpicc)
 #Add include pathes
-set(FLEUR_Fortran_FLAGS "-I$ENV{ELPA_MODULES} -I$ENV{EBROOTHDF5}/include -mkl")
 #Add linker stuff
-set(FLEUR_LIBRARIES "-L$ENV{ELPA_LIB};-lelpa_openmp;-lmkl_scalapack_lp64;-lmkl_blacs_intelmpi_lp64;-L$ENV{EBROOTHDF5}/lib;-lhdf5;-lhdf5_fortran")
+set(FLEUR_Fortran_FLAGS " -mkl")
\ No newline at end of file
+#set(FLEUR_Fortran_FLAGS "-I$ENV{ELPA_MODULES} -I$ENV{EBROOTHDF5}/include -mkl")
+#set(FLEUR_LIBRARIES "-L$ENV{ELPA_LIB};-lelpa_openmp;-lmkl_scalapack_lp64;-lmkl_blacs_intelmpi_lp64;-L$ENV{EBROOTHDF5}/lib;-lhdf5;-lhdf5_fortran")
+set(FLEUR_LIBRARIES "-lmkl_scalapack_lp64;-lmkl_blacs_intelmpi_lp64")

cmake/compilerflags.cmake

 #cmake file to set compiler flags for some of the known compilers
-
 if (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel")
    message("Intel Fortran detected")
    if (${CMAKE_Fortran_COMPILER_VERSION} VERSION_LESS "13.0.0.0")
@@ -9,9 +8,9 @@ if (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel")
    endif()     
    set(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -xHost -O4")
    set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -C -traceback -O0 -g")
-elseif(${CMAKE_Fortan_COMPILER_ID} MATCHES "PGI")
+elseif(${CMAKE_Fortran_COMPILER_ID} MATCHES "PGI")
    message("PGI Fortran detected")
-   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mp -Mr8 -Mr8intrinsics")
+   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mp -Mr8 -Mr8intrinsics  -Mcuda:kepler+ -ta:tesla:cuda7.5 -DUSE_STREAMS -DNUM_STREAMS=${N_STREAMS} -Minfo=accel -acc")
    set(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -fast -O3")
    set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -C -traceback -O0 -g -Mchkstk -Mchkptr")
 elseif(${CMAKE_Fortran_COMPILER_ID} MATCHES "XL")

cmake/machines.sh

@@ -15,13 +15,13 @@ function configure_machine(){
     if [ "$machine" = "JURECA" ] 
     then
 	echo "JURECA configuration used"
-	if module list 2>&1 |grep -q intel-para
+	if module list 2>&1 |grep -q -i intel
 	then
 	    echo "Intel toolchain used"
-	    if module list 2>&1| grep -q Python/2.7.12 &&
-               module list 2>&1| grep -q HDF5 &&
+	    if module list 2>&1| grep -q Python &&
                module list 2>&1| grep -q CMake &&
 	       module list 2>&1| grep -q ELPA
+               #module list 2>&1| grep -q HDF5 
 	    then
 		echo "All required modules load loaded"
 	    else
@@ -33,8 +33,19 @@ function configure_machine(){
 	elif module list 2>&1 |grep -q PGI
 	then
 	    echo "PGI toolchain used"
-	    echo "Needs to be configured"
-	    exit
+	    if module list 2>&1| grep -q MVAPICH2 &&
+               module list 2>&1| grep -q CMake &&
+	       [ -n "${MAGMA_ROOT}" ] &&
+	       [ -n "${XML2_ROOT}" ] 
+            then
+		echo "All required modules loaded, all variables set"
+	    else
+		echo "Please do at least"
+		echo "module load MVAPICH2 CMake"
+		echo "And set the variables XML2_ROOT and MAGMA_ROOT"
+		exit
+	    fi
+	    cp $DIR/cmake/JURECAGPU.cmake config.cmake
 	else
 	    echo "You have to load the correct modules for compiling"
 	    echo " a) intel-para"

diagonalization/eigen_diag.F90

@@ -111,6 +111,8 @@ CONTAINS
 #if 1==2
     !This is only needed for debugging
     print *,n_rank,lapw%nmat
+    print *,"SR:",n_size,n_rank
+    print *,mpi
     write(f,'(a,i0)') "a.",n_rank
     open(99,file=f)
     write(f,'(a,i0)') "b.",n_rank
@@ -126,7 +128,6 @@ CONTAINS
     CALL MPI_BARRIER(MPI_COMM_WORLD,err)
     close(99)
      close(98)
-    STOP 'DEBUG'
 #endif
     
     !
@@ -169,6 +170,9 @@ CONTAINS
        SELECT CASE (priv_select_solver(parallel))
 #ifdef CPP_ELPA
        CASE (diag_elpa)
+          CALL MPI_COMM_RANK(sub_comm,n,err)
+          write(*,*) "DIAG:",mpi%irank,sub_comm,n
+          write(*,*) "ELPA:",mpi%irank,lapw%nmat,ne_found,size(eig),ndim,ndim1
           IF (hamovlp%l_real) THEN
               CALL elpa_diag(lapw%nmat,SUB_COMM,hamOvlp%a_r,hamOvlp%b_r,zMat%z_r,eig,ne_found)
           ELSE
@@ -196,11 +200,13 @@ CONTAINS
           if (noco%l_ss) call juDFT_error("zsymsecloc not tested with noco%l_ss")
           if (input%gw>1) call juDFT_error("zsymsecloc not tested with input%gw>1")
           IF (l_real) THEN
+#ifndef __PGI
           CALL zsymsecloc(jsp,input,lapw,bkpt,atoms,kveclo, sym,l_zref,cell, dimension,matsize,ndim,&
                 jij,matind,nred,eig,ne_found,hamOvlp%a_r,hamOvlp%b_r,zMat%z_r)
        else
           CALL zsymsecloc(jsp,input,lapw,bkpt,atoms,kveclo, sym,l_zref,cell, dimension,matsize,ndim,&
                jij,matind,nred,eig,ne_found,hamOvlp%a_c,hamOvlp%b_c,zMat%z_c)
+#endif
        endif
        CASE (diag_lapack)
           CALL franza(dimension%nbasfcn,ndim, lapw%nmat,(l_zref.AND.(atoms%nlotot.EQ.0)),&

eigen/eigen.F90

@@ -314,7 +314,10 @@ CONTAINS
     endif
     eig_id=open_eig(&
          mpi%mpi_comm,dimension%nbasfcn,dimension%neigd,kpts%nkpt,dimension%jspd,atoms%lmaxd,&
-         atoms%nlod,atoms%ntype,atoms%nlotot,noco%l_noco,.true.,l_real,noco%l_soc,.false.,mpi%n_size,layers=vacuum%layers,nstars=vacuum%nstars,ncored=dimension%nstd,nsld=atoms%nat,nat=atoms%nat,l_dos=banddos%dos.or.input%cdinf,l_mcd=banddos%l_mcd,l_orb=banddos%l_orb)
+         atoms%nlod,atoms%ntype,atoms%nlotot,noco%l_noco,.true.,l_real,noco%l_soc,.false.,&
+         mpi%n_size,layers=vacuum%layers,nstars=vacuum%nstars,ncored=dimension%nstd,&
+         nsld=atoms%nat,nat=atoms%nat,l_dos=banddos%dos.or.input%cdinf,l_mcd=banddos%l_mcd,&
+         l_orb=banddos%l_orb)
 
     IF (l_real) THEN
        ALLOCATE ( hamOvlp%a_r(matsize), stat = err )

eigen_secvar/aline.F90

@@ -82,6 +82,9 @@ CONTAINS
 
 
     lhelp= MAX(lapw%nmat,(DIMENSION%neigd+2)*DIMENSION%neigd)
+#ifdef __PGI
+        STOP "CODE NOT WORKING WITH PGI"
+#else
     IF (l_real) THEN
        CALL read_eig(eig_id,nk,jsp,bk=bkpt,neig=ne,nv=lapw%nv(jsp),nmat=lapw%nmat, eig=eig,kveclo=kveclo,z=zMat%z_r)
        ALLOCATE ( h_r(DIMENSION%neigd,DIMENSION%neigd),s_r(DIMENSION%neigd,DIMENSION%neigd) )
@@ -89,14 +92,16 @@ CONTAINS
        ALLOCATE ( help_r(lhelp) )
     ELSE
        CALL read_eig(eig_id,nk,jsp,bk=bkpt,neig=ne,nv=lapw%nv(jsp),nmat=lapw%nmat, eig=eig,kveclo=kveclo,z=zMat%z_c)
-
        !     in outeig z is complex conjugated to make it usable for abcof. Here we 
        !                       first have to undo this  complex conjugation for the 
-       zMat%z_c = CONJG(zMat%z_c)    ! multiplication with a and b matrices.
+       ! multiplication with a and b matrices.
+
+       zmat%z_c=conjg(zmat%z_c)
        ALLOCATE ( h_c(DIMENSION%neigd,DIMENSION%neigd),s_c(DIMENSION%neigd,DIMENSION%neigd) )
        h_c = 0.0 ; s_c=0.0
        ALLOCATE ( help_r(lhelp) )
     ENDIF
+#endif
     !
     DO i = 1,ne
        IF (l_real) THEN

global/types.F90

@@ -759,8 +759,8 @@
         LOGICAL              :: l_real
         INTEGER              :: nbasfcn
         INTEGER              :: nbands
-        REAL,    ALLOCATABLE :: z_r(:,:) ! z_r(nbasfcn,nbands)
-        COMPLEX, ALLOCATABLE :: z_c(:,:) ! z_c(nbasfcn,nbands)
+        REAL,    POINTER :: z_r(:,:) ! z_r(nbasfcn,nbands)
+        COMPLEX, POINTER :: z_c(:,:) ! z_c(nbasfcn,nbands)
       END TYPE
 
       TYPE t_hamOvlp

init/ordstar.f

@@ -260,10 +260,10 @@ c       printout of representative vector vkrep of star
 c
           if (nkrep(ik) .lt. 1)
      +      write(iofile,'(1x,''WARNING: we have found no '',
-     +          ''k-point of star # '',i4, ''  inside irr BZ'')') ik
+     +          ''k-point of star no '',i4, ''  inside irr BZ'')') ik
           if (nkrep(ik) .gt. 1)
      +      write(iofile,'(1x,''WARNING: we have found more than '',
-     +      ''one k-point of star # '',i4, ''  inside irr BZ'')') ik
+     +      ''one k-point of star no '',i4, ''  inside irr BZ'')') ik
             write(iofile,'(1x,i4,3(1x,f10.7),1x,i4,10x,
      +         ''nkstar,vkrep(kpn),nkrep: repr k-point in irr BZ'',/)')
      +            ik,(vkrep(i1,ik),i1=1,3),nkrep(ik)
@@ -392,7 +392,7 @@ c
            if (ktest.ge.2) then
              write(iofile,'(/,'' result of ordering :'')')
              write(iofile,'(1x,i4,1x,i4,2x,
-     +              ''# of stars, # of k-points contained in them'')')
+     +              ''no of stars, no of k-points contained in them'')')
      +                                                   nstnew, isumkpt
            end if
 c
@@ -522,7 +522,7 @@ c
            if (ktest.ge.2) then
              write(iofile,'(/,'' result of ordering :'')')
              write(iofile,'(1x,i4,1x,i4,2x,
-     +              ''# of stars, # of k-points contained in them'')')
+     +              ''no of stars, no of k-points contained in them'')')
      +                                                   nstnew, isumkpt
            end if
 c
@@ -558,11 +558,11 @@ c --->  printout of representative vector vkrep of star
 c
           if (nkrep(ik) .lt. 1)
      +      write(iofile,'(1x,''WARNING: we have found no '',
-     +          ''k-point of star # '',i4, ''  inside irr BZ; '',
+     +          ''k-point of star no '',i4, ''  inside irr BZ; '',
      +          ''vkrep set to zero'')') ik
           if (nkrep(ik) .gt. 1)
      +      write(iofile,'(1x,''WARNING: we have found more than '',
-     +      ''one k-point of star # '',i4, ''  inside irr BZ; '',
+     +      ''one k-point of star no '',i4, ''  inside irr BZ; '',
      +      ''last vkrep shown'')') ik
             write(iofile,'(1x,i4,3(1x,f10.7),1x,i4,10x,
      +           ''nkstar,vkrep(kpn),nkrep: repr k-point in irr BZ'')')

math/util.F

@@ -549,9 +549,9 @@ c     np = max(0,int(log(n*0.001)/log(2d0))) should be a safe choice.
 
       INTEGER, INTENT(IN)  :: n,np
       INTEGER, INTENT(OUT) :: pnt(n)
-      REAL(8), INTENT(IN)  :: rarr(n)
-      REAL(8)              :: rarr1(n)
-      REAL(8)              :: ravg
+      REAL   , INTENT(IN)  :: rarr(n)
+      REAL                 :: rarr1(n)
+      REAL                 :: ravg
       INTEGER              :: pnt1(n),pnt2(n)
       INTEGER              :: n1,n2,i