Bugfixes and modifications for PGI compiler/MAGMA

cmake/compilerflags.cmake

@@ -13,6 +13,7 @@ elseif(${CMAKE_Fortran_COMPILER_ID} MATCHES "PGI")
    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mp -Mr8 -Mr8intrinsics  -Mcuda:kepler+ -ta:tesla:cuda7.5 -DUSE_STREAMS -DNUM_STREAMS=${N_STREAMS} -Minfo=accel -acc")
    set(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -fast -O3")
    set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -C -traceback -O0 -g -Mchkstk -Mchkptr")
+   set(FLEUR_LIBRARIES ${FLEUR_LIBRARIES} "-lstdc++;-L$ENV{MKLROOT}/lib/intel64;-lmkl_scalapack_lp64;-lmkl_intel_lp64;-lmkl_pgi_thread;-lmkl_core;-lmkl_blacs_intelmpi_lp64")
 elseif(${CMAKE_Fortran_COMPILER_ID} MATCHES "XL")
    message("IBM/BG Fortran detected")
    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qsmp=omp -qnosave -qarch=qp -qtune=qp -qrealsize=8 -qfixed -qsuppress=1520-022 -qessl") 

cmake/tests/test_MAGMA.cmake

@@ -6,5 +6,5 @@ LINK_LIBRARIES ${FLEUR_LIBRARIES}
 message("MAGMA Library found:${FLEUR_USE_MAGMA}")
 
 if (FLEUR_USE_MAGMA)
-   set(FLEUR_MPI_DEFINITIONS ${FLEUR_MPI_DEFINITIONS} "CPP_MAGMA")
+   set(FLEUR_DEFINITIONS ${FLEUR_DEFINITIONS} "CPP_MAGMA")
 endif() 

diagonalization/eigen_diag.F90

@@ -194,7 +194,12 @@ CONTAINS
 #endif
 #ifdef CPP_MAGMA
        CASE (diag_magma)
-          CALL magma_diag(lapw%nmat,eig,ne_found,hamOvlp%a_r,hamOvlp%b_r,zMat%z_r,hamOvlp%a_c,hamOvlp%b_c,zMat%z_c)
+          if (l_real) THEN
+             call juDFT_error("REAL diagonalization not implemented in magma.F90")
+          else
+             print *,"Start magma_diag"
+             CALL magma_diag(lapw%nmat,eig,ne_found,a_c=hamOvlp%a_c,b_c=hamOvlp%b_c,z_c=zMat%z_c)
+          endif
 #endif
        CASE (diag_lapack2)
           if (noco%l_ss) call juDFT_error("zsymsecloc not tested with noco%l_ss")

diagonalization/magma.F90

@@ -13,6 +13,7 @@ MODULE m_magma
   !**********************************************************
 CONTAINS
   SUBROUTINE magma_diag(nsize,eig,ne,a_r,b_r,z_r,a_c,b_c,z_c)
+    use m_packed_to_full
 #ifdef CPP_MAGMA
     use magma
 #endif    
@@ -62,17 +63,19 @@ CONTAINS
     if (l_real) THEN
        call packed_to_full(nsize,a_r,largea_r)
        call packed_to_full(nsize,b_r,largeb_r)
-       deallocate(a_r,b_r)
+       !deallocate(a_r,b_r)
     ELSE
        call packed_to_full(nsize,a_c,largea_c)
        call packed_to_full(nsize,b_c,largeb_c)
-       deallocate(a_c,b_c)
+       !deallocate(a_c,b_c)
     Endif
 
     if (l_real) call juDFT_error("REAL diagonalization not implemented in magma.F90")
 
     !Query the workspace size 
     allocate(work(1),rwork(1),iwork(1))
+    print *,"Magma workspace query"
+    call flush()
     call magmaf_zhegvdx_2stage_m(NGPU_CONST,1,MagmaVec,MagmaRangeI,MagmaLower,nsize,largea_c,nsize,largeb_c,nsize,&
          0.0,0.0,1,ne,mout,eigTemp,work,-1,rwork,-1,iwork,-1,err)
     lwork=work(1)
@@ -83,6 +86,8 @@ CONTAINS
     allocate(work(lwork),rwork(lrwork),iwork(liwork))
     if (err/=0) call juDFT_error("Failed to allocate workspaces",calledby="magma.F90")
     !Now the diagonalization
+    print *,"Magma diagonalization"
+    print *,nsize,shape(largea_c),shape(eigTemp),ne
     call magmaf_zhegvdx_2stage_m(NGPU_CONST,1,MagmaVec,MagmaRangeI,MagmaLower,nsize,largea_c,nsize,largeb_c,nsize,&
          0.0,0.0,1,ne,mout,eigTemp,work,lwork,rwork,lrwork,iwork,liwork,err)
     print*,"MAGMA info:",err
@@ -91,8 +96,9 @@ CONTAINS
 
     DO i = 1, ne
        eig(i) = eigTemp(i)
+       z_c(:nsize,i)=largea_c(:nsize,i)
     END DO
-    call judft_error("Eigenvectors are not calculated in MAGMA")
+    !call judft_error("Eigenvectors are not calculated in MAGMA")
 #endif
   END SUBROUTINE magma_diag
 END MODULE m_magma

init/strgn.f90

@@ -516,8 +516,8 @@ CONTAINS
           print *,stars%mx1, mxx1, stars%mx2, mxx2       
           CALL judft_error("BUG in strgn")
     endif
-    stars%mx1=mxx1
-    stars%mx2=mxx2
+    !stars%mx1=mxx1
+    !stars%mx2=mxx2
     !
     !--->    write /str0/ and /str1/ to unit 51
     !
@@ -894,7 +894,7 @@ CONTAINS
           ENDDO
        ENDDO
     ENDIF
-    if ( stars%mx1 .ne. mxx1 .or. stars%mx2 .ne. mxx2 .or. stars%mx3 .ne. mxx3 ) call &
+    if ( stars%mx1 < mxx1 .or. stars%mx2 < mxx2 .or. stars%mx3 < mxx3 ) call &
          judft_error("BUG 1 in strgen") 
     stars%ng2 = 2 ; stars%kv2 = 0 ; stars%ig2 = 0 ; igz = 0 ; stars%kimax2= 0 ; stars%igfft2 = 0
     stars%sk2 = 0.0 ; stars%pgfft2 = 0.0  ; stars%nstr2 = 0

main/fleur_init.F90

@@ -97,7 +97,7 @@
           !-t3e
           IF (mpi%irank.EQ.0) THEN
              CALL startXMLOutput()
-#ifndef  __TOS_BGQ__
+#if !(defined(__TOS_BGQ__)||defined(__PGI))
              !Do not open out-file on BlueGene
              OPEN (6,file='out',form='formatted',status='unknown')
 #endif