eig66_mpi.F90 20.8 KB
Newer Older
1 2
MODULE m_eig66_mpi
#include "juDFT_env.h"
3
  USE m_eig66_data
4
  USE m_types
5
#ifdef CPP_MPI
6
  use mpi
7
#endif
8 9
  IMPLICIT NONE
  PRIVATE
10
  PUBLIC open_eig,read_eig,write_eig,close_eig
11 12
CONTAINS

13 14 15
  SUBROUTINE priv_find_data(id,d)
    INTEGER,INTENT(IN)::id
    TYPE(t_data_mpi),POINTER:: d
16

17 18 19 20 21 22 23 24 25
    CLASS(t_data),POINTER   ::dp
    CALL eig66_find_data(dp,id)
    SELECT TYPE(dp)
    TYPE is (t_data_mpi)
       d=>dp
       CLASS default
       CALL judft_error("BUG: wrong datatype in eig66_mpi")
    END SELECT
  END SUBROUTINE priv_find_data
26 27


28
  SUBROUTINE open_eig(id,mpi_comm,nmat,neig,nkpts,jspins,lmax,nlo,ntype,create,l_real,l_soc,nlotot,l_noco,n_size_opt,l_dos,l_mcd,l_orb,filename,layers,nstars,ncored,nsld,nat)
29 30 31
    USE,INTRINSIC::iso_c_binding
    IMPLICIT NONE
    INTEGER, INTENT(IN) :: id,mpi_comm,nmat,neig,nkpts,jspins,nlo,ntype,lmax,nlotot
32
    LOGICAL, INTENT(IN) :: l_noco,create,l_real,l_soc
33
    INTEGER,INTENT(IN),OPTIONAL:: n_size_opt
34
    LOGICAL,INTENT(IN),OPTIONAL ::l_dos,l_mcd,l_orb
35 36 37 38
    CHARACTER(LEN=*),INTENT(IN),OPTIONAL :: filename
    INTEGER,INTENT(IN),OPTIONAL :: layers,nstars,ncored,nsld,nat
#ifdef CPP_MPI
    INTEGER:: isize,e,slot_size,local_slots
39
    INTEGER,PARAMETER::mcored=27 !there should not be more that 27 core states
40 41 42
    TYPE(t_data_MPI),POINTER :: d

    CALL priv_find_data(id,d)
43
    CALL eig66_data_storedefault(d,jspins,nkpts,nmat,neig,lmax,nlotot,nlo,ntype,l_real.and..not.l_soc,l_soc,l_dos,l_mcd,l_orb)
44 45 46 47 48 49

    IF (PRESENT(n_size_opt)) d%n_size=n_size_opt
    IF (ALLOCATED(d%pe_ev)) THEN
       IF (create) THEN
          d%neig_data=0
          d%eig_data=1E99
50
          d%w_iks_data=1E99
51
          if (d%l_real.and..not.l_soc) THEN
52 53 54 55
             d%zr_data=0.0
          else
             d%zc_data=0.0
          endif
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
          d%qal_data=0.0
          d%qvac_data=0.0
          d%qvlay_data=0.0
          d%qstars_data=0.0
          d%ksym_data=0.0
          d%jsym_data=0.0
          d%mcd_data=0.0
          d%qintsl_data=0.0
          d%qmtsl_data=0.0
          d%qmtp_data=0.0
          d%orbcomp_data=0.0
       ENDIF
       IF (PRESENT(filename)) CALL priv_readfromfile()
       RETURN !everything already done!
    ENDIF
71

72 73 74 75 76 77 78 79 80 81
    CALL timestart("create data spaces in ei66_mpi")
    CALL MPI_COMM_RANK(MPI_COMM,d%irank,e)
    CALL MPI_COMM_SIZE(MPI_COMM,isize,e)

    CALL create_maps(d,isize,nkpts,jspins,neig,d%n_size)
    local_slots=COUNT(d%pe_basis==d%irank)
    !Now create the windows

    !Window for neig
    slot_size=1
82
    CALL priv_create_memory(1,local_slots,d%neig_handle,d%neig_data)
83 84 85
    d%neig_data=0

    !The eigenvalues
86
    d%size_eig=neig
87
    CALL priv_create_memory(d%size_eig,local_slots,d%eig_handle,real_data_ptr=d%eig_data)
88
    d%eig_data=1E99
89 90 91 92
    !The w_iks
    CALL priv_create_memory(d%size_eig,local_slots,d%w_iks_handle,real_data_ptr=d%w_iks_data)
    d%w_iks_data=1E99

93 94 95 96
    !The eigenvectors
    local_slots=COUNT(d%pe_ev==d%irank)
    slot_size=nmat

97 98 99 100 101
    if (l_real.and..not.l_soc) THEN
       CALL priv_create_memory(slot_size,local_slots,d%zr_handle,real_data_ptr=d%zr_data)
    else
       CALL priv_create_memory(slot_size,local_slots,d%zc_handle,cmplx_data_ptr=d%zc_data)
    endif
102
    !Data for DOS etc
103
    IF (d%l_dos) THEN
104
       local_slots=COUNT(d%pe_basis==d%irank)
105 106 107
       CALL priv_create_memory(4*ntype*neig,local_slots,d%qal_handle,real_data_ptr=d%qal_data)
       CALL priv_create_memory(neig*2,local_slots,d%qvac_handle,real_data_ptr=d%qvac_data)
       CALL priv_create_memory(neig,local_slots,d%qis_handle,real_data_ptr=d%qis_data)
108 109
       CALL priv_create_memory(neig*max(1,layers)*2,local_slots,d%qvlay_handle,real_data_ptr=d%qvlay_data)
       CALL priv_create_memory(max(1,nstars)*neig*max(1,layers)*2,local_slots,d%qstars_handle,cmplx_data_ptr=d%qstars_data)
110 111 112
       CALL priv_create_memory(neig,local_slots,d%jsym_handle,d%jsym_data)
       CALL priv_create_memory(neig,local_slots,d%ksym_handle,d%ksym_data)
       IF (l_mcd) CALL priv_create_memory(3*ntype*mcored*neig,local_slots,d%mcd_handle,real_data_ptr=d%mcd_data)
113
       IF (l_orb) THEN
114 115 116 117
          CALL priv_create_memory(nsld*neig,local_slots,d%qintsl_handle,real_data_ptr=d%qintsl_data)
          CALL priv_create_memory(nsld*neig,local_slots,d%qmtsl_handle,real_data_ptr=d%qmtsl_data)
          CALL priv_create_memory(nat*neig,local_slots,d%qmtp_handle,real_data_ptr=d%qmtp_data)
          CALL priv_create_memory(23*nat*neig,local_slots,d%orbcomp_handle,real_data_ptr=d%orbcomp_data)
118 119 120 121 122 123 124
       ENDIF
    ELSE
       ALLOCATE(d%qal_data(1),d%qvac_data(1),d%qis_data(1),d%qvlay_data(1),d%qstars_data(1),&
            d%jsym_data(1),d%ksym_data(1),d%mcd_data(1),d%qintsl_data(1),d%qmtsl_data(1),&
            d%qmtp_data(1),d%orbcomp_data(1))
    ENDIF
    IF (PRESENT(filename).AND..NOT.create) CALL priv_readfromfile()
125
    CALL MPI_BARRIER(MPI_COMM,e)
126 127
    CALL timestop("create data spaces in ei66_mpi")
  CONTAINS
128
    SUBROUTINE priv_create_memory(slot_size,local_slots,handle,int_data_ptr,real_data_ptr,cmplx_data_ptr)
129 130
      IMPLICIT NONE
      INTEGER,INTENT(IN)           :: slot_size,local_slots
131 132 133
      INTEGER,POINTER,INTENT(OUT),OPTIONAL  :: int_data_ptr(:)
      REAL   ,POINTER,INTENT(OUT),OPTIONAL  :: real_data_ptr(:)
      COMPLEX,POINTER,INTENT(OUT),OPTIONAL  :: cmplx_data_ptr(:)
134
      INTEGER,INTENT(OUT)          :: handle
135
#ifdef CPP_MPI
136 137 138
      TYPE(c_ptr)::ptr
      INTEGER:: e
      INTEGER(MPI_ADDRESS_KIND) :: length
139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154
      INTEGER                   :: type_size

      length=0   
      IF (present(real_data_ptr)) THEN
          length=length+1
          CALL MPI_TYPE_SIZE(MPI_DOUBLE_PRECISION,type_size,e)
      ENDIF
      IF (present(cmplx_data_ptr)) THEN
          length=length+1
          CALL MPI_TYPE_SIZE(MPI_DOUBLE_COMPLEX,type_size,e)
      ENDIF
      IF (present(int_data_ptr)) THEN 
          length=length+1
          CALL MPI_TYPE_SIZE(MPI_INTEGER,type_size,e)
      ENDIF
      if (length.ne.1) call judft_error("Bug in eig66_mpi:create_memory") 
155
      length=MAX(1,slot_size*local_slots)
156
 
157 158 159 160
      length=length*type_size

      CALL MPI_ALLOC_MEM(length,MPI_INFO_NULL,ptr,e)
      IF (e.NE.0) CPP_error("Could not allocated MPI-Data in eig66_mpi")
161 162 163
	
      IF (present(real_data_ptr)) THEN	
      	CALL C_F_POINTER(ptr,real_data_ptr,(/length/type_size/))
164
      	CALL MPI_WIN_CREATE(real_data_ptr, length,slot_size*type_size,Mpi_INFO_NULL, MPI_COMM,handle, e)
165 166
      ELSEIF(present(int_data_ptr)) THEN
      	CALL C_F_POINTER(ptr,int_data_ptr,(/length/type_size/))
167
      	CALL MPI_WIN_CREATE(int_data_ptr, length,slot_size*type_size,Mpi_INFO_NULL, MPI_COMM,handle, e)
168 169
      ELSE
      	CALL C_F_POINTER(ptr,cmplx_data_ptr,(/length/type_size/))
170
      	CALL MPI_WIN_CREATE(cmplx_data_ptr, length,slot_size*type_size,Mpi_INFO_NULL, MPI_COMM,handle, e)
171
      ENDIF
172
#endif
173
    END SUBROUTINE priv_create_memory
174 175


176 177 178 179
    SUBROUTINE priv_readfromfile()
      USE m_eig66_DA,ONLY:open_eig_DA=>open_eig,read_eig_DA=>read_eig,close_eig_da=>close_eig
      INTEGER:: jspin,nk,i,ii,iii,nv,tmp_id
      REAL   :: wk,bk3(3),evac(2)
Daniel Wortmann's avatar
Daniel Wortmann committed
180
      REAL    :: eig(neig),w_iks(neig),ello(nlo,ntype),el(lmax,ntype)
181 182 183 184 185
      TYPE(t_zmat)::zmat
      zmat%l_real=d%l_real
      zmat%nbasfcn=nmat
      zmat%nbands=neig
      allocate(zmat%z_r(nmat,neig),zmat%z_c(nmat,neig))
186 187 188
      !only do this with PE=0
      IF (d%irank==0) THEN
         tmp_id=eig66_data_newid(DA_mode)
189
         IF (d%l_dos) CPP_error("Could not read DOS data")
190
         CALL open_eig_DA(tmp_id,nmat,neig,nkpts,jspins,lmax,nlo,ntype,nlotot,.FALSE.,.FALSE.,d%l_real,l_soc,.FALSE.,.FALSE.,filename)
191 192
         DO jspin=1,jspins
            DO nk=1,nkpts
193
               !CALL read_eig_DA(tmp_id,nk,jspin,nv,i,bk3,wk,ii,eig,w_iks,el,ello,evac,zmat=zmat)
194 195
               STOP "code no longer works"
                 ! CALL write_eig(id,nk,jspin,ii,ii,nv,nmat,bk3,wk,eig,w_iks,el,ello,evac,nlotot,zmat=zmat)
196
              ENDDO
197 198 199 200
         ENDDO
         CALL close_eig_DA(tmp_id)
      ENDIF
    END SUBROUTINE priv_readfromfile
201
#endif
202 203
  END SUBROUTINE open_eig
  SUBROUTINE close_eig(id,delete,filename)
204 205 206
    INTEGER,INTENT(IN)         :: id
    LOGICAL,INTENT(IN),OPTIONAL:: delete
    CHARACTER(LEN=*),INTENT(IN),OPTIONAL::filename
207 208
    TYPE(t_data_MPI),POINTER :: d
    CALL priv_find_data(id,d)
209

210 211
    IF (PRESENT(delete)) THEN
       IF (delete) WRITE(*,*) "No deallocation of memory implemented in eig66_mpi"
212
    ENDIF
213 214 215 216 217 218 219 220
    IF (PRESENT(filename)) CALL priv_writetofile()
  CONTAINS
    SUBROUTINE priv_writetofile()
      USE m_eig66_DA,ONLY:open_eig_DA=>open_eig,write_eig_DA=>write_eig,close_eig_DA=>close_eig
      IMPLICIT NONE

      INTEGER:: nlotot,nk,jspin,nv,i,ii,tmp_id
      REAL   :: wk,bk3(3),evac(2)
Daniel Wortmann's avatar
Daniel Wortmann committed
221
      REAL    :: eig(d%neig),w_iks(d%neig),ello(d%nlo,d%ntype),el(d%lmax,d%ntype)
222 223 224 225 226
      TYPE(t_zmat)::zmat
      zmat%l_real=d%l_real
      zmat%nbasfcn=d%nmat
      zmat%nbands=d%neig
      allocate(zmat%z_r(d%nmat,d%neig),zmat%z_c(d%nmat,d%neig))
227
      nlotot=d%nlotot
228

229 230 231
      IF (d%irank==0) THEN
         tmp_id=eig66_data_newid(DA_mode)
         IF (d%l_dos) CPP_error("Could not write DOS data")
232
         CALL open_eig_DA(tmp_id,d%nmat,d%neig,d%nkpts,d%jspins,d%lmax,d%nlo,d%ntype,d%nlotot,.FALSE.,.FALSE.,d%l_real,d%l_soc,.FALSE.,.FALSE.,filename)
233 234
         DO jspin=1,d%jspins
            DO nk=1,d%nkpts
235
               !CALL read_eig(id,nk,jspin,nv,i,bk3,wk,ii,eig,w_iks,el,ello,evac,zmat=zmat)
236 237
               stop "CODE no longer working"
               !CALL write_eig_DA(tmp_id,nk,jspin,ii,ii,nv,i,bk3,wk,eig,w_iks,el,ello,evac,nlotot,zmat=zmat)
238
            ENDDO
239
         ENDDO
240 241 242 243 244 245 246
         CALL close_eig_DA(tmp_id)
      ENDIF
      CALL eig66_remove_data(id)
    END SUBROUTINE priv_writetofile

  END SUBROUTINE close_eig

247
  SUBROUTINE read_eig(id,nk,jspin,neig,eig,w_iks,n_start,n_end,zmat)
248 249 250
    IMPLICIT NONE
    INTEGER, INTENT(IN)            :: id,nk,jspin
    INTEGER, INTENT(OUT),OPTIONAL  :: neig
Daniel Wortmann's avatar
Daniel Wortmann committed
251
    REAL,    INTENT(OUT),OPTIONAL  :: eig(:),w_iks(:)
252
    INTEGER, INTENT(IN),OPTIONAL   :: n_start,n_end
253
    TYPE(t_zmat),OPTIONAL  :: zmat
254 255

#ifdef CPP_MPI
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272
    INTEGER                   :: pe,tmp_size,e
    INTEGER(MPI_ADDRESS_KIND) :: slot
    INTEGER                   :: n1,n2,n3,n
    INTEGER,ALLOCATABLE       :: tmp_int(:)
    REAL,ALLOCATABLE          :: tmp_real(:)
    COMPLEX,ALLOCATABLE       :: tmp_cmplx(:)
    TYPE(t_data_MPI),POINTER :: d
    CALL priv_find_data(id,d)
    pe=d%pe_basis(nk,jspin)
    slot=d%slot_basis(nk,jspin)
    IF (PRESENT(neig))THEN
       CALL MPI_WIN_LOCK(MPI_LOCK_SHARED,pe,0,d%neig_handle,e)
       ! Get current values
       CALL  MPI_GET(neig,1,MPI_INTEGER,pe,slot,1,MPI_INTEGER,d%neig_handle,e)
       CALL MPI_WIN_UNLOCK(pe,d%neig_handle,e)

    ENDIF
Daniel Wortmann's avatar
Daniel Wortmann committed
273
    IF (PRESENT(eig).or.PRESENT(w_iks)) THEN
274
       ALLOCATE(tmp_real(d%size_eig))
Daniel Wortmann's avatar
Daniel Wortmann committed
275
       IF (PRESENT(eig)) THEN
276 277 278
          CALL MPI_WIN_LOCK(MPI_LOCK_SHARED,pe,0,d%eig_handle,e)
          CALL MPI_GET(tmp_real,d%size_eig,MPI_DOUBLE_PRECISION,pe,slot,d%size_eig,MPI_DOUBLE_PRECISION,d%eig_handle,e)
          CALL MPI_WIN_UNLOCK(pe,d%eig_handle,e)
Daniel Wortmann's avatar
Daniel Wortmann committed
279 280 281 282 283
          n1=1;n3=1;n2=SIZE(eig)
          IF (PRESENT(n_start)) n1=n_start
          IF (PRESENT(n_end)) n2=n_end
          eig(:n2-n1+1)=tmp_real(n1:n2)
       END IF
284 285 286 287 288 289
       IF (PRESENT(w_iks)) THEN
          CALL MPI_WIN_LOCK(MPI_LOCK_SHARED,pe,0,d%w_iks_handle,e)
          CALL MPI_GET(tmp_real,d%size_eig,MPI_DOUBLE_PRECISION,pe,slot,d%size_eig,MPI_DOUBLE_PRECISION,d%w_iks_handle,e)
          CALL MPI_WIN_UNLOCK(pe,d%w_iks_handle,e)
          w_iks=tmp_real(:size(w_iks))
       END IF
290 291 292
       DEALLOCATE(tmp_real)
    ENDIF

293 294
    IF (PRESENT(zmat)) THEN
       tmp_size=zmat%nbasfcn
295 296
       ALLOCATE(tmp_real(tmp_size))
       ALLOCATE(tmp_cmplx(tmp_size))
297
       DO n=1,zmat%nbands
298 299 300 301 302 303 304
          n1=n
          IF (PRESENT(n_start)) n1=n_start+n-1
          IF (PRESENT(n_end)) THEN
             IF (n1>n_end) CYCLE
          ENDIF
          slot=d%slot_ev(nk,jspin,n1)
          pe=d%pe_ev(nk,jspin,n1)
305 306
          
          if (zmat%l_real) THEN
307 308 309 310 311
             if (.not.d%l_real) THEN
                CALL MPI_WIN_LOCK(MPI_LOCK_SHARED,pe,0,d%zc_handle,e)
                CALL MPI_GET(tmp_cmplx,tmp_size,MPI_DOUBLE_COMPLEX,pe,slot,tmp_size,MPI_DOUBLE_COMPLEX,d%zc_handle,e)
                CALL MPI_WIN_UNLOCK(pe,d%zc_handle,e)
                !print *, nk,jspin,n1,"r PE:",pe," Slot: ",slot," Size:",tmp_size,tmp_cmplx(1)
312
                zmat%z_r(:,n)=REAL(tmp_cmplx)
313 314 315 316 317
             else
                CALL MPI_WIN_LOCK(MPI_LOCK_SHARED,pe,0,d%zr_handle,e)
                CALL MPI_GET(tmp_real,tmp_size,MPI_DOUBLE_PRECISION,pe,slot,tmp_size,MPI_DOUBLE_PRECISION,d%zr_handle,e)
                CALL MPI_WIN_UNLOCK(pe,d%zr_handle,e)
                !print *, nk,jspin,n1,"r PE:",pe," Slot: ",slot," Size:",tmp_size,tmp_real(1)
318
                zmat%z_r(:,n)=tmp_real
319
             endif
320
          ELSE
321
             if (d%l_real) call judft_error("Could not read complex data, only real data is stored",calledby="eig66_mpi%read_eig")
322 323 324 325
             CALL MPI_WIN_LOCK(MPI_LOCK_SHARED,pe,0,d%zc_handle,e)
             CALL MPI_GET(tmp_cmplx,tmp_size,MPI_DOUBLE_COMPLEX,pe,slot,tmp_size,MPI_DOUBLE_COMPLEX,d%zc_handle,e)
             CALL MPI_WIN_UNLOCK(pe,d%zc_handle,e)
             !print *, nk,jspin,n1,"r PE:",pe," Slot: ",slot," Size:",tmp_size,tmp_cmplx(1)
326
             zmat%z_c(:,n)=tmp_cmplx
Daniel Wortmann's avatar
Daniel Wortmann committed
327
          ENDIF
328 329
       ENDDO
    ENDIF
330 331

#endif
332 333
  END SUBROUTINE read_eig

334
  SUBROUTINE write_eig(id,nk,jspin,neig,neig_total,eig,w_iks,n_size,n_rank,zmat)
335 336
    INTEGER, INTENT(IN)          :: id,nk,jspin
    INTEGER, INTENT(IN),OPTIONAL :: n_size,n_rank
337 338
    INTEGER, INTENT(IN),OPTIONAL :: neig,neig_total
    REAL,    INTENT(IN),OPTIONAL :: eig(:),w_iks(:)
339
    TYPE(t_mat),INTENT(IN),OPTIONAL :: zmat
340 341

#ifdef CPP_MPI
342 343
    INTEGER                   :: pe,tmp_size,e
    INTEGER(MPI_ADDRESS_KIND) :: slot
344
    INTEGER                   :: n1,n2,n3,n,nn
345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364
    INTEGER,ALLOCATABLE       :: tmp_int(:)
    REAL,ALLOCATABLE          :: tmp_real(:)
    COMPLEX,ALLOCATABLE       :: tmp_cmplx(:)
    LOGICAL                   :: acc
    TYPE(t_data_MPI),POINTER :: d

    CALL priv_find_data(id,d)

    pe=d%pe_basis(nk,jspin)
    slot=d%slot_basis(nk,jspin)
    !write the number of eigenvalues values
    IF (PRESENT(neig_total)) THEN
       CALL MPI_WIN_LOCK(MPI_LOCK_EXCLUSIVE,pe,0,d%neig_handle,e)
       ALLOCATE(tmp_int(1))
       tmp_int(1)=neig_total
       CALL MPI_PUT(tmp_int,1,MPI_INTEGER,pe,slot,1,MPI_INTEGER,d%neig_handle,e)
       CALL MPI_WIN_UNLOCK(pe,d%neig_handle,e)
       DEALLOCATE(tmp_int)
    ENDIF

Daniel Wortmann's avatar
Daniel Wortmann committed
365
    IF (PRESENT(eig).OR.PRESENT(w_iks)) THEN
366 367
       ALLOCATE(tmp_real(d%size_eig))
       tmp_real=1E99
Daniel Wortmann's avatar
Daniel Wortmann committed
368 369 370 371 372 373 374 375 376 377
       if (PRESENT(EIG)) THEN
          n1=1;n3=1
          IF (PRESENT(n_rank)) n1=n_rank+1
          IF (PRESENT(n_size)) n3=n_size
          n2=SIZE(eig)*n3+n1-1
          nn=1
          DO n=n1,min(n2,d%size_eig),n3
             tmp_real(n)=eig(nn)
             nn=nn+1
          ENDDO
378 379 380 381 382 383 384
          CALL MPI_WIN_LOCK(MPI_LOCK_EXCLUSIVE,pe,0,d%eig_handle,e)
          IF (n3.ne.1) THEN
             CALL MPI_ACCUMULATE(tmp_real,d%size_eig,MPI_DOUBLE_PRECISION,pe,slot,d%size_eig,MPI_DOUBLE_PRECISION,MPI_MIN,d%eig_handle,e)
          ELSE
             CALL MPI_PUT(tmp_real,d%size_eig,MPI_DOUBLE_PRECISION,pe,slot,d%size_eig,MPI_DOUBLE_PRECISION,d%eig_handle,e)
          ENDIF
          CALL MPI_WIN_UNLOCK(pe,d%eig_handle,e)
Daniel Wortmann's avatar
Daniel Wortmann committed
385
       END if
386 387 388 389 390 391
       IF (PRESENT(w_iks)) THEN
          tmp_real(:size(w_iks))=w_iks
          CALL MPI_WIN_LOCK(MPI_LOCK_EXCLUSIVE,pe,0,d%w_iks_handle,e)
          CALL MPI_PUT(tmp_real,d%size_eig,MPI_DOUBLE_PRECISION,pe,slot,d%size_eig,MPI_DOUBLE_PRECISION,d%w_iks_handle,e)
          CALL MPI_WIN_UNLOCK(pe,d%w_iks_handle,e)
       END IF
392 393
       DEALLOCATE(tmp_real)
    ENDIF
394
    IF (PRESENT(zmat)) THEN
395
       tmp_size=zmat%matsize1
396 397
       ALLOCATE(tmp_real(tmp_size))
       ALLOCATE(tmp_cmplx(tmp_size))
398
       DO n=1,zmat%matsize2
399 400 401
          n1=n-1
          IF (PRESENT(n_size)) n1=n_size*n1
          IF (PRESENT(n_rank)) n1=n1+n_rank
402
          IF (n1+1>size(d%slot_ev,3)) EXIT
403 404 405
          slot=d%slot_ev(nk,jspin,n1+1)
          pe=d%pe_ev(nk,jspin,n1+1)
          !print *, "PE:",pe," Slot: ",slot," Size:",tmp_size,tmp_real(1)
406
          IF (zmat%l_real) THEN
407
             if (.not.d%l_real) THEN
408
                tmp_cmplx=zmat%data_r(:,n)
409 410 411 412
                CALL MPI_WIN_LOCK(MPI_LOCK_EXCLUSIVE,pe,0,d%zc_handle,e)
                CALL MPI_PUT(tmp_cmplx,tmp_size,MPI_DOUBLE_COMPLEX,pe,slot,tmp_size,MPI_DOUBLE_COMPLEX,d%zc_handle,e)
                CALL MPI_WIN_UNLOCK(pe,d%zc_handle,e)
             else
413
                tmp_real=zmat%data_r(:,n)
414 415 416 417
                CALL MPI_WIN_LOCK(MPI_LOCK_EXCLUSIVE,pe,0,d%zr_handle,e)
                CALL MPI_PUT(tmp_real,tmp_size,MPI_DOUBLE_PRECISION,pe,slot,tmp_size,MPI_DOUBLE_PRECISION,d%zr_handle,e)
                CALL MPI_WIN_UNLOCK(pe,d%zr_handle,e)
             endif
418
          ELSE
419
             if (d%l_real) CALL juDFT_error("Could not write complex data to file prepared for real data",calledby="eig66_mpi%write_eig")
420
             tmp_cmplx=zmat%data_c(:,n)
421 422 423
             CALL MPI_WIN_LOCK(MPI_LOCK_EXCLUSIVE,pe,0,d%zc_handle,e)
             CALL MPI_PUT(tmp_cmplx,tmp_size,MPI_DOUBLE_COMPLEX,pe,slot,tmp_size,MPI_DOUBLE_COMPLEX,d%zc_handle,e)
             CALL MPI_WIN_UNLOCK(pe,d%zc_handle,e)
424
          ENDIF
425 426
       ENDDO
    ENDIF
427 428

#endif
429 430
  END SUBROUTINE write_eig

431
#ifdef CPP_MPI
432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448
  SUBROUTINE priv_put_data(pe,slot,DATA,handle)
    IMPLICIT NONE
    INTEGER,INTENT(IN)  :: pe,slot
    CLASS(*),INTENT(IN) :: DATA(:)
    INTEGER,INTENT(IN)  :: handle

    INTEGER             :: len,e
    INTEGER,ALLOCATABLE :: int_tmp(:)
    REAL,ALLOCATABLE    :: real_tmp(:)
    COMPLEX,ALLOCATABLE:: cmplx_tmp(:)
    INCLUDE 'mpif.h'
    len=SIZE(DATA)
    SELECT TYPE(DATA)
    TYPE IS (INTEGER)
       ALLOCATE(int_tmp(len))
       int_tmp=DATA
       CALL MPI_WIN_LOCK(MPI_LOCK_EXCLUSIVE,pe,0,handle,e)
449
       CALL MPI_PUT(int_tmp,len,MPI_INTEGER,pe,int(slot,MPI_ADDRESS_KIND),len,MPI_INTEGER,handle,e)
450 451 452 453 454
       CALL MPI_WIN_UNLOCK(pe,handle,e)
    TYPE is (REAL)
       ALLOCATE(real_tmp(len))
       real_tmp=DATA
       CALL MPI_WIN_LOCK(MPI_LOCK_EXCLUSIVE,pe,0,handle,e)
455
       CALL MPI_PUT(real_tmp,len,MPI_DOUBLE_PRECISION,pe,int(slot,MPI_ADDRESS_KIND),len,MPI_DOUBLE_PRECISION,handle,e)
456 457 458 459 460
       CALL MPI_WIN_UNLOCK(pe,handle,e)
    TYPE is (COMPLEX)
       ALLOCATE(cmplx_tmp(len))
       cmplx_tmp=DATA
       CALL MPI_WIN_LOCK(MPI_LOCK_EXCLUSIVE,pe,0,handle,e)
461
       CALL MPI_PUT(cmplx_tmp,len,MPI_DOUBLE_COMPLEX,pe,int(slot,MPI_ADDRESS_KIND),len,MPI_DOUBLE_COMPLEX,handle,e)
462 463 464 465
       CALL MPI_WIN_UNLOCK(pe,handle,e)
    END SELECT
  END SUBROUTINE priv_put_data

466
  SUBROUTINE priv_get_data(pe,slot,len,handle,idata,rdata,cdata)
467 468
    IMPLICIT NONE
    INTEGER,INTENT(IN)  :: pe,slot,len
469 470 471
    INTEGER,INTENT(OUT),optional :: iDATA(len)
    REAL,INTENT(OUT),optional    :: rDATA(len)
    COMPLEX,INTENT(OUT),optional :: cDATA(len)
472 473 474 475 476 477 478 479
    INTEGER,INTENT(IN)  :: handle

    INTEGER             :: e
    INTEGER,ALLOCATABLE :: int_tmp(:)
    REAL,ALLOCATABLE    :: real_tmp(:)
    COMPLEX,ALLOCATABLE:: cmplx_tmp(:)
    INCLUDE 'mpif.h'

480
    IF (present(idata)) THEN
481 482
       ALLOCATE(int_tmp(len))
       CALL MPI_WIN_LOCK(MPI_LOCK_SHARED,pe,0,handle,e)
483
       CALL MPI_GET(int_tmp,len,MPI_INTEGER,pe,int(slot,MPI_ADDRESS_KIND),len,MPI_INTEGER,handle,e)
484
       CALL MPI_WIN_UNLOCK(pe,handle,e)
485 486
       iDATA=int_tmp
    ELSE IF (PRESENT(rdata)) THEN
487 488
       ALLOCATE(real_tmp(len))
       CALL MPI_WIN_LOCK(MPI_LOCK_SHARED,pe,0,handle,e)
489
       CALL MPI_GET(real_tmp,len,MPI_DOUBLE_PRECISION,pe,int(slot,MPI_ADDRESS_KIND),len,MPI_DOUBLE_PRECISION,handle,e)
490
       CALL MPI_WIN_UNLOCK(pe,handle,e)
491 492
       rDATA=real_tmp
    ELSE IF (PRESENT(cdata)) THEN
493 494
       ALLOCATE(cmplx_tmp(len))
       CALL MPI_WIN_LOCK(MPI_LOCK_SHARED,pe,0,handle,e)
495
       CALL MPI_GET(cmplx_tmp,len,MPI_DOUBLE_COMPLEX,pe,int(slot,MPI_ADDRESS_KIND),len,MPI_DOUBLE_COMPLEX,handle,e)
496
       CALL MPI_WIN_UNLOCK(pe,handle,e)
497 498 499 500 501
       cDATA=cmplx_tmp
    ELSE
       call judft_error("BUG in priv_get_data")
    ENDIF

502
  END SUBROUTINE priv_get_data
503
#endif
504

505
#ifdef CPP_MPI
506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543
  SUBROUTINE create_maps(d,isize,nkpts,jspins,neig,n_size)
    IMPLICIT NONE
    TYPE(t_data_MPI),INTENT(INOUT):: d
    INTEGER,INTENT(IN):: isize,nkpts,jspins,neig,n_size

    INTEGER:: nk,j,n1,n2,n,pe,n_members
    INTEGER::used(0:isize)

    ALLOCATE(d%pe_basis(nkpts,jspins),d%slot_basis(nkpts,jspins))
    ALLOCATE(d%pe_ev(nkpts,jspins,neig),d%slot_ev(nkpts,jspins,neig))

    !basis contains a total of nkpts*jspins entries
    d%pe_basis=-1
    d%pe_ev=-1
    used=0
    n_members=isize/n_size !no of k-points in parallel
    DO j=1,jspins
       DO nk=1,nkpts
          n1=nk+(j-1)*nkpts-1
          pe=MOD(n1,n_members)*n_size
          d%pe_basis(nk,j)=pe
          d%slot_basis(nk,j)=used(pe)
          used(pe)=used(pe)+1
       ENDDO
    ENDDO
    used=0
    DO n=1,neig
       DO j=1,jspins
          DO nk=1,nkpts
             n1=nk+(j-1)*nkpts-1
             !eigenvectors have more entries
             pe=MOD(n1,n_members)*n_size+MOD(n,n_size)
             d%pe_ev(nk,j,n)=pe
             d%slot_ev(nk,j,n)=used(pe)
             used(pe)=used(pe)+1
          ENDDO
       ENDDO
    ENDDO
544

545
  END SUBROUTINE create_maps
546 547 548 549


#endif
END MODULE m_eig66_mpi