Commit ec0c5fa5 authored by Matthias Redies's avatar Matthias Redies

drop coulomb io replace with MPI_IBcast

parent 574c8ea7
......@@ -102,14 +102,16 @@ CONTAINS
CALL timestop("generation of mixed basis")
if(mpi%irank == 0) then
CALL open_hybinp_io2(mpdata, fi%hybinp, hybdat, fi%input, fi%atoms, fi%sym%invs)
CALL coulombmatrix(mpi, fi, mpdata, hybdat, xcpot)
call close_hybinp_io2()
endif
call hybmpi%barrier()
CALL open_hybinp_io2(mpdata, fi%hybinp, hybdat, fi%input, fi%atoms, fi%sym%invs)
if(.not. allocated(hybdat%coul)) allocate(hybdat%coul(fi%kpts%nkpt))
do i =1,fi%kpts%nkpt
call hybdat%coul(i)%alloc(fi, mpdata%num_radbasfn, mpdata%n_g)
enddo
if(mpi%irank == 0) CALL coulombmatrix(mpi, fi, mpdata, hybdat, xcpot)
do i =1,fi%kpts%nkpt
call hybdat%coul(i)%mpi_ibc(fi, hybmpi)
enddo
CALL hf_init(eig_id, mpdata, fi, hybdat)
CALL timestop("Preparation for hybrid functionals")
......
......@@ -910,7 +910,6 @@ CONTAINS
call timestart("loop bla")
DO ikpt = 1, fi%kpts%nkpt
! initialize arrays to 0
call hybdat%coul(ikpt)%alloc(fi, mpdata%num_radbasfn, mpdata%n_g)
call hybdat%coul(ikpt)%init()
! unpack coulomb into coulhlp
......@@ -1143,21 +1142,8 @@ CONTAINS
ic2 = indx1 + mpdata%n_g(ikpt)
hybdat%coul(ikpt)%pmtir_c(:ic2*(ic2 + 1)/2) = packmat(hybdat%coul(ikpt)%mtir_c(:ic2, :ic2))
end if
call timestart("write coulomb_spm")
if (fi%sym%invs) THEN
CALL write_coulomb_spm_r(ikpt, hybdat%coul(ikpt)%mt1(:, :, :, :), hybdat%coul(ikpt)%mt2_r(:, :, :, :), &
hybdat%coul(ikpt)%mt3_r(:, :, :), hybdat%coul(ikpt)%pmtir_r(:))
else
call write_coulomb_spm_c(ikpt, hybdat%coul(ikpt)%mt1(:, :, :, :), hybdat%coul(ikpt)%mt2_c(:, :, :, :), &
hybdat%coul(ikpt)%mt3_c(:, :, :), hybdat%coul(ikpt)%pmtir_c(:))
endif
call timestop("write coulomb_spm")
END DO ! ikpt
call timestop("loop bla")
do ikpt = 1, fi%kpts%nkpt
call hybdat%coul(ikpt)%free()
enddo
CALL timestop("Coulomb matrix setup")
END SUBROUTINE coulombmatrix
......
......@@ -102,7 +102,7 @@ CONTAINS
! local scalars
INTEGER :: iband, iband1, jq, iq
INTEGER :: i, ierr
INTEGER :: j
INTEGER :: j, iq_p
INTEGER :: n, n1, n2, nn, nn2
INTEGER :: nkqpt
INTEGER :: ok
......@@ -127,17 +127,6 @@ CONTAINS
REAL, ALLOCATABLE :: cprod_vv_r(:, :, :), carr3_vv_r(:, :, :)
COMPLEX, ALLOCATABLE :: cprod_vv_c(:, :, :), carr3_vv_c(:, :, :)
REAL :: coulomb_mt1(maxval(mpdata%num_radbasfn) - 1, maxval(mpdata%num_radbasfn) - 1, 0:maxval(fi%hybinp%lcutm1), fi%atoms%ntype)
REAL :: coulomb_mt2_r(maxval(mpdata%num_radbasfn) - 1, -maxval(fi%hybinp%lcutm1):maxval(fi%hybinp%lcutm1), 0:maxval(fi%hybinp%lcutm1) + 1, fi%atoms%nat)
REAL :: coulomb_mt3_r(maxval(mpdata%num_radbasfn) - 1, fi%atoms%nat, fi%atoms%nat)
COMPLEX :: coulomb_mt2_c(maxval(mpdata%num_radbasfn) - 1, -maxval(fi%hybinp%lcutm1):maxval(fi%hybinp%lcutm1), 0:maxval(fi%hybinp%lcutm1) + 1, fi%atoms%nat)
COMPLEX :: coulomb_mt3_c(maxval(mpdata%num_radbasfn) - 1, fi%atoms%nat, fi%atoms%nat)
REAL :: coulomb_mtir_r(((maxval(fi%hybinp%lcutm1) + 1)**2*fi%atoms%nat + maxval(mpdata%n_g))* &
((maxval(fi%hybinp%lcutm1) + 1)**2*fi%atoms%nat + maxval(mpdata%n_g) + 1)/2)
COMPLEX :: coulomb_mtir_c(((maxval(fi%hybinp%lcutm1) + 1)**2*fi%atoms%nat + maxval(mpdata%n_g))* &
((maxval(fi%hybinp%lcutm1) + 1)**2*fi%atoms%nat + maxval(mpdata%n_g) + 1)/2)
LOGICAL :: occup(fi%input%neig)
CALL timestart("valence exchange calculation")
......@@ -183,19 +172,7 @@ CONTAINS
IF (hybdat%nbasm(iq) /= n) call judft_error('error hybdat%nbasm')
nn = n*(n + 1)/2
! read in coulomb matrix from direct access file coulomb
IF (mat_ex%l_real) THEN
CALL read_coulomb_spm_r(fi%kpts%bkp(iq), coulomb_mt1, coulomb_mt2_r, coulomb_mt3_r, coulomb_mtir_r)
ELSE
CALL read_coulomb_spm_c(fi%kpts%bkp(iq), coulomb_mt1, coulomb_mt2_c, coulomb_mt3_c, coulomb_mtir_c)
END IF
IF (fi%kpts%bkp(iq) /= iq) THEN
IF ((fi%kpts%bksym(iq) > fi%sym%nop) .and. (.not. mat_ex%l_real)) THEN
coulomb_mt2_c = conjg(coulomb_mt2_c)
coulomb_mtir_c = conjg(coulomb_mtir_c)
END IF
END IF
iq_p = fi%kpts%bkp(iq)
IF (mat_ex%l_real) THEN
CALL wavefproducts_inv(fi, ik, z_k, iq, jsp, lapw, hybdat, mpdata, nococonv, nkqpt, cprod_vv_r)
......@@ -239,20 +216,27 @@ CONTAINS
END IF
! calculate exchange matrix at iq
call timestart("exchange matrix")
! finish coulomb bcast
call hybdat%coul(iq_p)%mpi_wait()
DO n1 = 1, hybdat%nbands(ik)
DO iband = 1, hybdat%nobd(nkqpt, jsp)
cdum = wl_iks(1 + iband - 1, nkqpt)*conjg(phase_vv(iband, n1))/n_q(jq)
call timestart("sparse matrix products")
IF (mat_ex%l_real) THEN
carr1_v_r(:n) = 0
CALL spmvec_invs(fi%atoms, mpdata, fi%hybinp, hybdat, iq, coulomb_mt1, coulomb_mt2_r, coulomb_mt3_r, &
coulomb_mtir_r, cprod_vv_r(:n, iband, n1), carr1_v_r(:n))
CALL spmvec_invs(fi%atoms, mpdata, fi%hybinp, hybdat, iq, hybdat%coul(iq_p)%mt1, hybdat%coul(iq_p)%mt2_r, &
hybdat%coul(iq_p)%mt3_r, hybdat%coul(iq_p)%pmtir_r, cprod_vv_r(:n, iband, n1), carr1_v_r(:n))
ELSE
carr1_v_c(:n) = 0
CALL spmvec_noinvs(fi%atoms, mpdata, fi%hybinp, hybdat, iq, coulomb_mt1, coulomb_mt2_c, coulomb_mt3_c, &
coulomb_mtir_c, cprod_vv_c(:n, iband, n1), carr1_v_c(:n))
if(fi%kpts%bksym(iq) > fi%sym%nop) then
CALL spmvec_noinvs(fi%atoms, mpdata, fi%hybinp, hybdat, iq, hybdat%coul(iq_p)%mt1, conjg(hybdat%coul(iq_p)%mt2_c),&
hybdat%coul(iq_p)%mt3_c, conjg(hybdat%coul(iq_p)%pmtir_c), cprod_vv_c(:n, iband, n1), carr1_v_c(:n))
else
CALL spmvec_noinvs(fi%atoms, mpdata, fi%hybinp, hybdat, iq, hybdat%coul(iq_p)%mt1, hybdat%coul(iq_p)%mt2_c,&
hybdat%coul(iq_p)%mt3_c, hybdat%coul(iq_p)%pmtir_c, cprod_vv_c(:n, iband, n1), carr1_v_c(:n))
endif
END IF
call timestop("sparse matrix products")
......
......@@ -10,105 +10,9 @@ module m_io_hybinp
use m_types
implicit none
!private
integer, save :: id_olap, id_z, id_v_x, id_coulomb, id_coulomb_spm
integer, save :: id_olap, id_z, id_v_x
!public:: open_hybinp_io,read_cmt,write_cmt
contains
SUBROUTINE open_hybinp_io2(mpdata, hybinp, hybdat, input, atoms, l_real)
IMPLICIT NONE
type(t_mpdata), intent(in) :: mpdata
TYPE(t_hybinp), INTENT(IN) :: hybinp
TYPE(t_hybdat), INTENT(IN) :: hybdat
TYPE(t_input), INTENT(IN) :: input
TYPE(t_atoms), INTENT(IN) :: atoms
LOGICAL, INTENT(IN) :: l_real
INTEGER:: irecl_coulomb
! if the sparse matrix technique is used, several entries of the
! matrix vanish so that the size of each entry is smaller
irecl_coulomb = (atoms%ntype*(maxval(hybinp%lcutm1) + 1)*(maxval(mpdata%num_radbasfn) - 1)**2 &
+ atoms%nat*(maxval(hybinp%lcutm1) + 2)*(2*maxval(hybinp%lcutm1) + 1)*(maxval(mpdata%num_radbasfn) - 1) &
+ (maxval(mpdata%num_radbasfn) - 1)*atoms%nat**2 &
+ ((maxval(hybinp%lcutm1) + 1)**2*atoms%nat + maxval(mpdata%n_g)) &
*((maxval(hybinp%lcutm1) + 1)**2*atoms%nat + maxval(mpdata%n_g) + 1)/2)*8
if(.not. l_real) irecl_coulomb = irecl_coulomb*2
OPEN(unit=778, file='coulomb1', form='unformatted', access='direct', recl=irecl_coulomb)
id_coulomb_spm = 778
END SUBROUTINE open_hybinp_io2
subroutine close_hybinp_io2
implicit none
close(778)
end subroutine close_hybinp_io2
subroutine write_coulomb(nk, l_real, coulomb)
implicit none
complex, intent(in) :: coulomb(:)
integer, intent(in) :: nk
logical, intent(in) :: l_real
if(l_real) THEN
write(id_coulomb, rec=nk) real(coulomb)
else
write(id_coulomb, rec=nk) coulomb
end if
end subroutine write_coulomb
subroutine write_coulomb_spm_r(nk, coulomb_mt1, coulomb_mt2, coulomb_mt3, coulomb_mtir)
implicit none
real, intent(in) :: coulomb_mt1(:, :, :, :)
real, intent(in) :: coulomb_mt2(:, :, :, :), coulomb_mt3(:, :, :)
real, intent(in) :: coulomb_mtir(:)
integer, intent(in) :: nk
write(id_coulomb_spm, rec=nk) coulomb_mt1, coulomb_mt2, coulomb_mt3, coulomb_mtir
end subroutine write_coulomb_spm_r
subroutine write_coulomb_spm_c(nk, coulomb_mt1, coulomb_mt2, coulomb_mt3, coulomb_mtir)
implicit none
real, intent(in) :: coulomb_mt1(:, :, :, :)
complex, intent(in) :: coulomb_mt2(:, :, :, :), coulomb_mt3(:, :, :)
complex, intent(in) :: coulomb_mtir(:)
integer, intent(in) :: nk
write(id_coulomb_spm, rec=nk) coulomb_mt1, coulomb_mt2, coulomb_mt3, coulomb_mtir
end subroutine write_coulomb_spm_c
subroutine read_coulomb_spm_r(nk, coulomb_mt1, coulomb_mt2, coulomb_mt3, coulomb_mtir)
implicit none
real, intent(out) :: coulomb_mt1(:, :, :, :)
real, intent(out) :: coulomb_mt2(:, :, :, :), coulomb_mt3(:, :, :)
real, intent(out) :: coulomb_mtir(:)
integer, intent(in) :: nk
!print *, "read coulomb",nk,size(coulomb_mt1),size(coulomb_mt2),size(coulomb_mt3),size(coulomb_mtir)
read(id_coulomb_spm, rec=nk) coulomb_mt1, coulomb_mt2, coulomb_mt3, coulomb_mtir
end subroutine read_coulomb_spm_r
subroutine read_coulomb_spm_c(nk, coulomb_mt1, coulomb_mt2, coulomb_mt3, coulomb_mtir)
implicit none
real, intent(out) :: coulomb_mt1(:, :, :, :)
complex, intent(out) :: coulomb_mt2(:, :, :, :), coulomb_mt3(:, :, :)
complex, intent(out) :: coulomb_mtir(:)
integer, intent(in) :: nk
read(id_coulomb_spm, rec=nk) coulomb_mt1, coulomb_mt2, coulomb_mt3, coulomb_mtir
end subroutine read_coulomb_spm_c
subroutine read_coulomb_r(nk, coulomb)
implicit none
real, intent(out) :: coulomb(:)
integer, intent(in) :: nk
read(id_coulomb, rec=nk) coulomb
end subroutine read_coulomb_r
subroutine read_coulomb_c(nk, coulomb)
implicit none
complex, intent(out) :: coulomb(:)
integer, intent(in) :: nk
read(id_coulomb, rec=nk) coulomb
end subroutine read_coulomb_c
subroutine read_olap(olap, rec, nbasfcn)
implicit none
TYPE(t_mat), INTENT(INOUT):: olap
......
......@@ -126,6 +126,7 @@ SUBROUTINE rdmft(eig_id,mpi,fi,enpara,stars,&
INTEGER, ALLOCATABLE :: n_q(:)
LOGICAL, ALLOCATABLE :: enabledConstraints(:)
type(t_hybmpi) :: hybmpi
complex :: c_phase(fi%input%neig)
......@@ -378,9 +379,13 @@ SUBROUTINE rdmft(eig_id,mpi,fi,enpara,stars,&
CALL mixedbasis(fi%atoms,fi%kpts,fi%input,fi%cell,xcpot,fi%mpinp,mpdata,fi%hybinp, hybdat,enpara,mpi,vTot, iterHF)
CALL open_hybinp_io2(mpdata, fi%hybinp,hybdat,fi%input,fi%atoms,fi%sym%invs)
!CALL open_hybinp_io2(mpdata, fi%hybinp,hybdat,fi%input,fi%atoms,fi%sym%invs)
CALL coulombmatrix(mpi, fi, mpdata, hybdat, xcpot)
if(mpi%irank == 0) CALL coulombmatrix(mpi, fi, mpdata, hybdat, xcpot)
call hybmpi%copy_mpi(mpi)
do i =1,fi%kpts%nkpt
call hybdat%coul(i)%mpi_ibc(fi, hybmpi)
enddo
CALL hf_init(eig_id,mpdata,fi,hybdat)
......
......@@ -9,10 +9,14 @@ MODULE m_types_hybdat
REAL, ALLOCATABLE :: mtir_r(:, :), pmtir_r(:)
COMPLEX, ALLOCATABLE :: mt2_c(:, :, :, :), mt3_c(:, :, :)
COMPLEX, ALLOCATABLE :: mtir_c(:, :), pmtir_c(:)
integer :: bcast_req(5)
logical :: bcast_finished = .False.
contains
procedure :: init => t_coul_init
procedure :: alloc => t_coul_alloc
procedure :: free => t_coul_free
procedure :: init => t_coul_init
procedure :: alloc => t_coul_alloc
procedure :: free => t_coul_free
procedure :: mpi_ibc => t_coul_mpi_ibc
procedure :: mpi_wait => t_coul_mpi_wait
end type t_coul
TYPE t_hybdat
......@@ -58,10 +62,70 @@ MODULE m_types_hybdat
END TYPE t_hybdat
contains
subroutine t_coul_mpi_wait(coul)
use m_judft
use mpi
implicit none
class(t_coul) :: coul
integer :: ierr, i
if(.not. coul%bcast_finished) then
do i = 1,size(coul%bcast_req)
call MPI_WAIT(coul%bcast_req(i), MPI_STATUS_IGNORE, ierr)
if(ierr /= 0) call judft_error("Error in MPI_wait for coul%bcast_req no. " // int2str(i))
enddo
coul%bcast_finished = .True.
endif
end subroutine t_coul_mpi_wait
subroutine t_coul_mpi_ibc(coul, fi, hybmpi)
use m_types_fleurinput
use m_types_hybmpi
use m_judft
use mpi
implicit none
class(t_coul) :: coul
type(t_fleurinput), intent(in) :: fi
type(t_hybmpi), intent(in) :: hybmpi
integer :: ierr
integer, parameter :: root = 0
call MPI_IBcast(coul%mt1, size(coul%mt1), MPI_DOUBLE_PRECISION, root, hybmpi%comm, coul%bcast_req(1), ierr)
if(ierr /= 0) call judft_error("MPI_IBcast of coul%mt1 failed")
if (fi%sym%invs) THEN
call MPI_IBcast(coul%mt2_r, size(coul%mt2_r), MPI_DOUBLE_PRECISION, root, hybmpi%comm, coul%bcast_req(2), ierr)
if(ierr /= 0) call judft_error("MPI_IBcast of coul%mt2_r failed")
call MPI_IBcast(coul%mt3_r, size(coul%mt3_r), MPI_DOUBLE_PRECISION, root, hybmpi%comm, coul%bcast_req(3), ierr)
if(ierr /= 0) call judft_error("MPI_IBcast of coul%mt3_r failed")
call MPI_IBcast(coul%mtir_r, size(coul%mtir_r), MPI_DOUBLE_PRECISION, root, hybmpi%comm, coul%bcast_req(4), ierr)
if(ierr /= 0) call judft_error("MPI_IBcast of coul%mtir_r failed")
call MPI_IBcast(coul%pmtir_r, size(coul%pmtir_r), MPI_DOUBLE_PRECISION, root, hybmpi%comm, coul%bcast_req(5), ierr)
if(ierr /= 0) call judft_error("MPI_IBcast of coul%pmtir_r failed")
else
call MPI_IBcast(coul%mt2_c, size(coul%mt2_c), MPI_DOUBLE_COMPLEX , root, hybmpi%comm, coul%bcast_req(2), ierr)
if(ierr /= 0) call judft_error("MPI_IBcast of coul%mt2_r failed")
call MPI_IBcast(coul%mt3_c, size(coul%mt3_c), MPI_DOUBLE_COMPLEX , root, hybmpi%comm, coul%bcast_req(3), ierr)
if(ierr /= 0) call judft_error("MPI_IBcast of coul%mt3_r failed")
call MPI_IBcast(coul%mtir_c, size(coul%mtir_c), MPI_DOUBLE_COMPLEX , root, hybmpi%comm, coul%bcast_req(4), ierr)
if(ierr /= 0) call judft_error("MPI_IBcast of coul%mtir_r failed")
call MPI_IBcast(coul%pmtir_c, size(coul%pmtir_c), MPI_DOUBLE_COMPLEX , root, hybmpi%comm, coul%bcast_req(5), ierr)
if(ierr /= 0) call judft_error("MPI_IBcast of coul%pmtir_r failed")
endif
coul%bcast_finished = .False.
end subroutine t_coul_mpi_ibc
subroutine t_coul_free(coul)
implicit none
class(t_coul), intent(inout) :: coul
if(allocated(coul%mt1)) deallocate(coul%mt1)
if(allocated(coul%mt2_r)) deallocate(coul%mt2_r)
if(allocated(coul%mt3_r)) deallocate(coul%mt3_r)
if(allocated(coul%mtir_r)) deallocate(coul%mtir_r)
......@@ -93,33 +157,49 @@ contains
endif
if (fi%sym%invs) THEN
if(.not. allocated(coul%mt2_r)) allocate(coul%mt2_r(maxval(num_radbasfn) - 1,&
-maxval(fi%hybinp%lcutm1):maxval(fi%hybinp%lcutm1),&
0:maxval(fi%hybinp%lcutm1) + 1, fi%atoms%nat), stat=info)
if(info /= 0) call judft_error("Can't allocate coul%mt2_r")
if(.not. allocated(coul%mt2_r)) then
allocate(coul%mt2_r(maxval(num_radbasfn) - 1,&
-maxval(fi%hybinp%lcutm1):maxval(fi%hybinp%lcutm1),&
0:maxval(fi%hybinp%lcutm1) + 1, fi%atoms%nat), stat=info)
if(info /= 0) call judft_error("Can't allocate coul%mt2_r")
endif
if(.not. allocated(coul%mt3_r)) allocate(coul%mt3_r(maxval(num_radbasfn) - 1,fi%atoms%nat, fi%atoms%nat), stat=info)
if(info /= 0) call judft_error("Can't allocate coul%mt3_r")
if(.not. allocated(coul%mt3_r)) then
allocate(coul%mt3_r(maxval(num_radbasfn) - 1,fi%atoms%nat, fi%atoms%nat), stat=info)
if(info /= 0) call judft_error("Can't allocate coul%mt3_r")
endif
if(.not. allocated(coul%mtir_r)) allocate(coul%mtir_r(ic + maxval(n_g),ic + maxval(n_g)), stat=info)
if(info /= 0) call judft_error("Can't allocate coul%mtir_r")
if(.not. allocated(coul%mtir_r)) then
allocate(coul%mtir_r(ic + maxval(n_g),ic + maxval(n_g)), stat=info)
if(info /= 0) call judft_error("Can't allocate coul%mtir_r")
endif
if(.not. allocated(coul%pmtir_r)) allocate(coul%pmtir_r(idum), stat=info)
if(info /= 0) call judft_error("Can't allocate coul%pmtir_r")
if(.not. allocated(coul%pmtir_r)) then
allocate(coul%pmtir_r(idum), stat=info)
if(info /= 0) call judft_error("Can't allocate coul%pmtir_r")
endif
else
if(.not. allocated(coul%mt2_c)) allocate(coul%mt2_c(maxval(num_radbasfn) - 1,&
-maxval(fi%hybinp%lcutm1):maxval(fi%hybinp%lcutm1), &
0:maxval(fi%hybinp%lcutm1) + 1, fi%atoms%nat), stat=info)
if(info /= 0) call judft_error("Can't allocate coul%mt2_c")
if(.not. allocated(coul%mt3_c)) allocate(coul%mt3_c(maxval(num_radbasfn) - 1, fi%atoms%nat, fi%atoms%nat), stat=info)
if(info /= 0) call judft_error("Can't allocate coul%mt3_c")
if(.not. allocated(coul%mtir_c)) allocate(coul%mtir_c(ic + maxval(n_g), ic + maxval(n_g)), stat=info)
if(info /= 0) call judft_error("Can't allocate coul%mtir_c")
if(.not. allocated(coul%pmtir_c)) allocate(coul%pmtir_c(idum), stat=info)
if(info /= 0) call judft_error("Can't allocate coul%pmtir_c")
if(.not. allocated(coul%mt2_c)) then
allocate(coul%mt2_c(maxval(num_radbasfn) - 1,&
-maxval(fi%hybinp%lcutm1):maxval(fi%hybinp%lcutm1), &
0:maxval(fi%hybinp%lcutm1) + 1, fi%atoms%nat), stat=info)
if(info /= 0) call judft_error("Can't allocate coul%mt2_c")
endif
if(.not. allocated(coul%mt3_c)) then
allocate(coul%mt3_c(maxval(num_radbasfn) - 1, fi%atoms%nat, fi%atoms%nat), stat=info)
if(info /= 0) call judft_error("Can't allocate coul%mt3_c")
endif
if(.not. allocated(coul%mtir_c)) then
allocate(coul%mtir_c(ic + maxval(n_g), ic + maxval(n_g)), stat=info)
if(info /= 0) call judft_error("Can't allocate coul%mtir_c")
endif
if(.not. allocated(coul%pmtir_c)) then
allocate(coul%pmtir_c(idum), stat=info)
if(info /= 0) call judft_error("Can't allocate coul%pmtir_c")
endif
endif
end subroutine t_coul_alloc
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment