Difference between revisions of "VASP make"

From UFRC
Jump to navigation Jump to search
(Created page with "=== Makefile for VASP 5 === <source lang=make> .SUFFIXES: .inc .f .f90 .F SUFFIX=.f90 FC=mpif90 FCL=$(FC) FFLAGS = -free -lowercase -assume byterecl OFLAG=-O3 OFLAG_HIGH ...")
 
Line 2: Line 2:
 
<source lang=make>
 
<source lang=make>
 
.SUFFIXES: .inc .f .f90 .F
 
.SUFFIXES: .inc .f .f90 .F
 +
#-----------------------------------------------------------------------
 +
# Makefile for Intel Fortran compiler at UF HPC
 +
#-----------------------------------------------------------------------
 +
 +
# all CPP processed fortran files have the extension .f90
 
SUFFIX=.f90
 
SUFFIX=.f90
  
FC=mpif90
+
#-----------------------------------------------------------------------
 +
# fortran compiler and linker
 +
#-----------------------------------------------------------------------
 +
FC=mpif90  
 
FCL=$(FC)
 
FCL=$(FC)
  
FFLAGS = -free -lowercase -assume byterecl
+
CPP_=fpp -f_com=no -free -w0 $*.F $*$(SUFFIX)
  
OFLAG=-O3
 
OFLAG_HIGH = $(OFLAG)
 
OBJ_HIGH =
 
OBJ_NOOPT =
 
DEBUG  = -free -O0
 
INLINE = $(OFLAG)
 
 
#-----------------------------------------------------------------------
 
#-----------------------------------------------------------------------
# additional options for CPP in parallel version (see also above):
+
# possible options for CPP:
# NGZhalf              charge density  reduced in Z direction
+
# NGXhalf            charge density  reduced in X direction
# wNGZhalf              gamma point only reduced in Z direction
+
# wNGXhalf            gamma point only reduced in X direction
# scaLAPACK            use scaLAPACK (usually slower on 100 Mbit Net)
+
# avoidalloc          avoid ALLOCATE if possible
 +
# PGF90              work around some for some PGF90 / IFC bugs
 +
# CACHE_SIZE          1000 for PII,PIII, 5000 for Athlon, 8000-12000 P4, PD
 +
# RPROMU_DGEMV        use DGEMV instead of DGEMM in RPRO (depends on used BLAS)
 +
# RACCMU_DGEMV        use DGEMV instead of DGEMM in RACC (depends on used BLAS)
 +
# tbdyn                MD package of Tomas  Bucko
 
#-----------------------------------------------------------------------
 
#-----------------------------------------------------------------------
# Define these when using MKL BLAS?
 
#    -DRPROMU_DGEMV  -DRACCMU_DGEMV
 
#-----------------------------------------------------------------------
 
CPP_ = ./preprocess <$*.F | /usr/bin/cpp -P -C -traditional >$*$(SUFFIX)
 
  
CPP = $(CPP_) -DHOST=\"LinuxUFHPC\" \
+
CPP_=fpp -f_com=no -free -w0 $*.F $*$(SUFFIX)
 +
CPP = $(CPP_) \
 +
-DHOST=\"Linux-UFHPC\" \
 
-DMPI \
 
-DMPI \
 
-DIFC \
 
-DIFC \
 
-DPGF90 \
 
-DPGF90 \
-Dkind8 \
 
-DNGZhalf \
 
 
-Davoidalloc \
 
-Davoidalloc \
-DMPI_BLOCK=8000 \
+
-DMPI_BLOCK=8000 \
 
-DCACHE_SIZE=4000 \
 
-DCACHE_SIZE=4000 \
-DscaLAPACK
+
-Duse_collective \
 +
-DscaLAPACK  
 +
 
 
#-----------------------------------------------------------------------
 
#-----------------------------------------------------------------------
# MKL 10.X
+
# optimization
# set -DRPROMU_DGEMV -DRACCMU_DGEMV in the CPP lines
+
# we have tested whether higher optimisation improves performance
#
+
# -axK  SSE1 optimization, but also generate code executable on all mach.
MKLDIR = -L/opt/intel/mkl/10.0.2.018/lib/em64t
+
#      xK improves performance somewhat on XP, and a is required in order
MKLCORE = -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
+
#      to run the code on older Athlons as well
MKL    = $(MKLDIR) $(MKLCORE)
+
# -xW  SSE2 optimization
 +
# -axW  SSE2 optimization, but also generate code executable on all mach.
 +
# -tpp6 P3 optimization
 +
# -tpp7 P4 optimization
 
#-----------------------------------------------------------------------
 
#-----------------------------------------------------------------------
  
#-----------------------------------------------------------------------
 
# GOTO BLAS Library (Slightly faster than MKL)
 
# http://www.cs.utexas.edu/users/kgoto/signup_first.html
 
#
 
BLAS = -L/opt/lib/goto/1.26 lgoto_penrynp-r1.26
 
#-----------------------------------------------------------------------
 
  
 +
FFLAGS =  -free -names lowercase -assume byterecl
 +
 +
OFLAG=-O2 -xSSE2 -axSSE4.2,SSE4.1,SSSE3,SSE3 -unroll-aggressive -warn general
 +
 +
OFLAG_HIGH = $(OFLAG)
 +
OBJ_HIGH =
 +
OBJ_NOOPT =
 +
DEBUG  = -free -O0
 +
INLINE = $(OFLAG)
 +
 +
#------------------------------MKL--------------------------------------
 +
 +
MKLDIR    = $(HPC_MKL_DIR)
 +
MKLLIBS  = -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
 +
MKLLIBDIR = $(HPC_MKL_DIR)/lib/intel64
 +
MKL      = -L$(MKLLIBDIR) $(MKLLIBS)
 
#-----------------------------------------------------------------------
 
#-----------------------------------------------------------------------
# ScaLaPack + BLACS
+
SCA       = -lmkl_scalapack_lp64
#
+
BLACS     = -lmkl_blacs_openmpi_lp64
SCA   = -L/opt/lib/scalapack/intel10/mvapich -lscalapack
 
BLACS = -L/opt/lib/blacs/intel10/mvapich -lBlacsF77Init -lBlacsCInit -lBlacs
 
 
 
 
SCALAPACK = $(SCA) $(BLACS)
 
SCALAPACK = $(SCA) $(BLACS)
#-----------------------------------------------------------------------
 
 
 
#-----------------------------------------------------------------------
 
#-----------------------------------------------------------------------
 
# MKL FFTs w/ FFTW wrappers
 
# MKL FFTs w/ FFTW wrappers
 
#
 
#
FFTLIB  = -lfftw3xf_intel
+
FFTLIB  = -lfftw3xf
 
 
# Bonifide FFTW 3.1.2
 
#FFTLIB  = -L/opt/intel/fftw/3.1.2/lib -lfftw3
 
 
#-----------------------------------------------------------------------
 
#-----------------------------------------------------------------------
 +
# Miscellaneous VASP routines
 +
#
 +
MISC = -Llib -ldmy
  
 
#-----------------------------------------------------------------------
 
#-----------------------------------------------------------------------
# Miscellaneous VASP routines
+
LIBS = $(FFTLIB) $(SCALAPACK) $(BLAS) $(MKL) $(MISC)
 +
RPATH  = -Wl,-rpath -Wl,$(MKLDIR)
 +
LINK  = -shared-intel $(RPATH)
  
MISC = -L../vasp.5.lib -ldmy
 
 
#-----------------------------------------------------------------------
 
#-----------------------------------------------------------------------
 +
FFT3D = fftmpi_map.o fftmpiw.o fftw3d.o fft3dlib.o
  
LIBS = $(MISC) $(FFTLIB) $(SCALAPACK) $(BLAS) $(MKL)
 
 
LINK  = -i_dynamic -Wl,-rpath -Wl,$(MKLDIR)
 
 
#-----------------------------------------------------------------------
 
#-----------------------------------------------------------------------
 
# general rules and compile lines
 
# general rules and compile lines
 
#-----------------------------------------------------------------------
 
#-----------------------------------------------------------------------
LPD_OBJS = ../vasp.5.lib/linpack_double.o
+
BASIC=   symmetry.o symlib.o  lattlib.o  random.o  
FFT_OBJS = fftmpi.o fftmpi_map.o fftw3d.o fft3dlib.o 
 
 
 
BASIC = symmetry.o symlib.o  lattlib.o  random.o  
 
  
SOURCE = base.o    mpi.o      smart_allocate.o      xml.o  \
+
SOURCE= base.o    mpi.o      smart_allocate.o      xml.o  \
 
         constant.o jacobi.o  main_mpi.o  scala.o  \
 
         constant.o jacobi.o  main_mpi.o  scala.o  \
         asa.o      lattice.o  poscar.o  ini.o       xclib.o     xclib_grad.o \
+
         asa.o      lattice.o  poscar.o  ini.o mgrid.o  xclib.o vdw_nl.o  xclib_grad.o \
         radial.o  pseudo.o  mgrid.o    gridq.o    ebs.o  \
+
         radial.o  pseudo.o  gridq.o    ebs.o  \
         mkpoints.o wave.o    wave_mpi.o  wave_high.o  \
+
         mkpoints.o wave.o    wave_mpi.o  wave_high.o  spinsym.o \
 
         $(BASIC)  nonl.o    nonlr.o    nonl_high.o dfast.o    choleski2.o \
 
         $(BASIC)  nonl.o    nonlr.o    nonl_high.o dfast.o    choleski2.o \
 
         mix.o      hamil.o    xcgrad.o  xcspin.o    potex1.o  potex2.o  \
 
         mix.o      hamil.o    xcgrad.o  xcspin.o    potex1.o  potex2.o  \
         metagga.o constrmag.o cl_shift.o relativistic.o LDApU.o \
+
         constrmag.o cl_shift.o relativistic.o LDApU.o \
         paw_base.o egrad.o    pawsym.o  pawfock.o  pawlhf.o   paw.o  \
+
         paw_base.o metagga.o  egrad.o    pawsym.o  pawfock.o  pawlhf.o   rhfatm.o  hyperfine.o paw.
         mkpoints_full.o      charge.o  dipol.o    pot.o \
+
o  \
 +
         mkpoints_full.o      charge.o  Lebedev-Laikov.o  stockholder.o dipol.o    pot.o \
 
         dos.o      elf.o      tet.o      tetweight.o hamil_rot.o \
 
         dos.o      elf.o      tet.o      tetweight.o hamil_rot.o \
         steep.o    chain.o    dyna.o    sphpro.o    us.o  core_rel.o \
+
         chain.o    dyna.o    k-proj.o    sphpro.o    us.o  core_rel.o \
 
         aedens.o  wavpre.o  wavpre_noio.o broyden.o \
 
         aedens.o  wavpre.o  wavpre_noio.o broyden.o \
         dynbr.o    rmm-diis.o reader.o  writer.o  tutor.o xml_writer.o \
+
         dynbr.o    hamil_high.o  rmm-diis.o reader.o  writer.o  tutor.o xml_writer.o \
 
         brent.o    stufak.o  fileio.o  opergrid.o stepver.o  \
 
         brent.o    stufak.o  fileio.o  opergrid.o stepver.o  \
         chgloc.o  fast_aug.o fock.o     mkpoints_change.o sym_grad.o \
+
         chgloc.o  fast_aug.o fock_multipole.o  fock.o mkpoints_change.o sym_grad.o \
         mymath.o  internals.o dimer_heyden.o dvvtrajectory.o vdwforcefield.o \
+
         mymath.o  internals.o npt_dynamics.o  dynconstr.o dimer_heyden.o dvvtrajectory.o vdwforcef
         hamil_high.o nmr.o   force.o \
+
ield.o \
        pead.o    subrot.o  subrot_scf.o pwlhf.o gw_model.o optreal.o   davidson.o \
+
         nmr.o     pead.o    subrot.o  subrot_scf.o \
 +
        force.o    pwlhf.o   gw_model.o optreal.o steep.o    davidson.o  david_inner.o \
 
         electron.o rot.o  electron_all.o shm.o    pardens.o  paircorrection.o \
 
         electron.o rot.o  electron_all.o shm.o    pardens.o  paircorrection.o \
 
         optics.o  constr_cell_relax.o  stm.o    finite_diff.o elpol.o    \
 
         optics.o  constr_cell_relax.o  stm.o    finite_diff.o elpol.o    \
 
         hamil_lr.o rmm-diis_lr.o  subrot_cluster.o subrot_lr.o \
 
         hamil_lr.o rmm-diis_lr.o  subrot_cluster.o subrot_lr.o \
 
         lr_helper.o hamil_lrf.o  elinear_response.o ilinear_response.o \
 
         lr_helper.o hamil_lrf.o  elinear_response.o ilinear_response.o \
         linear_optics.o linear_response.o  \
+
         linear_optics.o \
 
         setlocalpp.o  wannier.o electron_OEP.o electron_lhf.o twoelectron4o.o \
 
         setlocalpp.o  wannier.o electron_OEP.o electron_lhf.o twoelectron4o.o \
         ratpol.o screened_2e.o wave_cacher.o chi_base.o wpot.o local_field.o \
+
         mlwf.o    ratpol.o screened_2e.o wave_cacher.o chi_base.o wpot.o \
         ump2.o bse.o acfdt.o chi.o sydmat.o  
+
        local_field.o ump2.o ump2kpar.o fcidump.o ump2no.o \
 +
         bse_te.o bse.o acfdt.o chi.o sydmat.o dmft.o \
 +
        rmm-diis_mlr.o  linear_response_NMR.o wannier_interpol.o linear_response.o
  
INC =
+
INCS = -I$(MKLDIR)/include/fftw
 
+
vasp: $(SOURCE) $(FFT3D) $(INC) main.o  
vasp: $(SOURCE) $(FFT_OBJS) $(INC) main.o  
+
rm -f vasp
        rm -f vasp
+
$(FCL) -o vasp main.o  $(SOURCE)   $(FFT3D) $(LINK) $(LIBS)
        $(FCL) -o vasp main.o  $(SOURCE) $(LPD_OBJS) $(FFT_OBJS) $(LIBS) $(LINK)
+
makeparam: $(SOURCE) $(FFT3D) makeparam.o main.F $(INC)
makeparam: $(SOURCE) $(FFT_OBJS) makeparam.o main.F $(INC)
+
$(FCL) -o makeparam  $(LINK) makeparam.o $(SOURCE) $(FFT3D) $(LIBS)
        $(FCL) -o makeparam  $(LINK) makeparam.o $(SOURCE) $(FFT_OBJS) $(LIBS)
 
 
zgemmtest: zgemmtest.o base.o random.o $(INC)
 
zgemmtest: zgemmtest.o base.o random.o $(INC)
        $(FCL) -o zgemmtest $(LINK) zgemmtest.o random.o base.o $(LIBS)
+
$(FCL) -o zgemmtest $(LINK) zgemmtest.o random.o base.o $(LIBS)
 
dgemmtest: dgemmtest.o base.o random.o $(INC)
 
dgemmtest: dgemmtest.o base.o random.o $(INC)
        $(FCL) -o dgemmtest $(LINK) dgemmtest.o random.o base.o $(LIBS)  
+
$(FCL) -o dgemmtest $(LINK) dgemmtest.o random.o base.o $(LIBS)  
ffttest: base.o smart_allocate.o mpi.o mgrid.o random.o ffttest.o $(FFT_OBJS) $(INC)
+
ffttest: base.o smart_allocate.o mpi.o mgrid.o random.o ffttest.o $(FFT3D) $(INC)
        $(FCL) -o ffttest $(LINK) ffttest.o mpi.o mgrid.o random.o smart_allocate.o base.o $(FFT_OBJS) $(LIBS)
+
$(FCL) -o ffttest $(LINK) ffttest.o mpi.o mgrid.o random.o smart_allocate.o base.o $(FFT3D) $
kpoints: $(SOURCE) $(FFT_OBJS) makekpoints.o main.F $(INC)
+
(LIBS)
        $(FCL) -o kpoints $(LINK) makekpoints.o $(SOURCE) $(FFT_OBJS) $(LIBS)
+
kpoints: $(SOURCE) $(FFT3D) makekpoints.o main.F $(INC)
 +
$(FCL) -o kpoints $(LINK) makekpoints.o $(SOURCE) $(FFT3D) $(LIBS)
  
clean:
+
clean:
        -rm -f *.g *.f *.f90 *.o *.L *.mod ; touch *.F
+
-rm -f *.g *.f *.o *.L *.mod ; touch *.F
  
 
main.o: main$(SUFFIX)
 
main.o: main$(SUFFIX)
        $(FC) $(FFLAGS)$(DEBUG)  $(INCS) -c main$(SUFFIX)
+
$(FC) $(FFLAGS)$(DEBUG)  $(INCS) -c main$(SUFFIX)
 
xcgrad.o: xcgrad$(SUFFIX)
 
xcgrad.o: xcgrad$(SUFFIX)
        $(FC) $(FFLAGS) $(INLINE)  $(INCS) -c xcgrad$(SUFFIX)
+
$(FC) $(FFLAGS) $(INLINE)  $(INCS) -c xcgrad$(SUFFIX)
 
xcspin.o: xcspin$(SUFFIX)
 
xcspin.o: xcspin$(SUFFIX)
        $(FC) $(FFLAGS) $(INLINE)  $(INCS) -c xcspin$(SUFFIX)
+
$(FC) $(FFLAGS) $(INLINE)  $(INCS) -c xcspin$(SUFFIX)
  
 
makeparam.o: makeparam$(SUFFIX)
 
makeparam.o: makeparam$(SUFFIX)
        $(FC) $(FFLAGS)$(DEBUG)  $(INCS) -c makeparam$(SUFFIX)
+
$(FC) $(FFLAGS)$(DEBUG)  $(INCS) -c makeparam$(SUFFIX)
  
 
makeparam$(SUFFIX): makeparam.F main.F  
 
makeparam$(SUFFIX): makeparam.F main.F  
 +
#
 +
# MIND: I do not have a full dependency list for the include
 +
# and MODULES: here are only the minimal basic dependencies
 +
# if one strucuture is changed then touch_dep must be called
 +
# with the corresponding name of the structure
 
#
 
#
 
base.o: base.inc base.F
 
base.o: base.inc base.F
Line 152: Line 170:
 
setex.o: setexm.inc setex.F
 
setex.o: setexm.inc setex.F
 
pseudo.o: pseudo.inc pseudo.F
 
pseudo.o: pseudo.inc pseudo.F
poscar.o: poscar.inc poscar.F
 
 
mkpoints.o: mkpoints.inc mkpoints.F
 
mkpoints.o: mkpoints.inc mkpoints.F
wave.o: wave.inc wave.F
+
wave.o: wave.F
 
nonl.o: nonl.inc nonl.F
 
nonl.o: nonl.inc nonl.F
 
nonlr.o: nonlr.inc nonlr.F
 
nonlr.o: nonlr.inc nonlr.F
  
 
$(OBJ_HIGH):
 
$(OBJ_HIGH):
        $(CPP)
+
$(CPP)
        $(FC) $(FFLAGS) $(OFLAG_HIGH) $(INCS) -c $*$(SUFFIX)
+
$(FC) $(FFLAGS) $(OFLAG_HIGH) $(INCS) -c $*$(SUFFIX)
 
$(OBJ_NOOPT):
 
$(OBJ_NOOPT):
        $(CPP)
+
$(CPP)
        $(FC) $(FFLAGS) $(INCS) -c $*$(SUFFIX)
+
$(FC) $(FFLAGS) $(INCS) -c $*$(SUFFIX)
  
 
fft3dlib_f77.o: fft3dlib_f77.F
 
fft3dlib_f77.o: fft3dlib_f77.F
        $(CPP)
+
$(CPP)
        $(F77) $(FFLAGS_F77) -c $*$(SUFFIX)
+
$(F77) $(FFLAGS_F77) -c $*$(SUFFIX)
  
 
.F.o:
 
.F.o:
        $(CPP)
+
$(CPP)
        $(FC) $(FFLAGS) $(OFLAG) $(INCS) -c $*$(SUFFIX)
+
$(FC) $(FFLAGS) $(OFLAG) $(INCS) -c $*$(SUFFIX)
 
.F$(SUFFIX):
 
.F$(SUFFIX):
        $(CPP)
+
$(CPP)
 
$(SUFFIX).o:
 
$(SUFFIX).o:
        $(FC) $(FFLAGS) $(OFLAG) $(INCS) -c $*$(SUFFIX)
+
$(FC) $(FFLAGS) $(OFLAG) $(INCS) -c $*$(SUFFIX)
  
 
# special rules
 
# special rules
 
#-----------------------------------------------------------------------
 
#-----------------------------------------------------------------------
 +
# these special rules have been tested for ifc.11 and ifc.12 only
  
 
fft3dlib.o : fft3dlib.F
 
fft3dlib.o : fft3dlib.F
        $(CPP)
+
$(CPP)
        $(FC) -free -lowercase -O2 -c $*$(SUFFIX)
+
$(FC) -FR -lowercase -O2 -c $*$(SUFFIX)
 
fft3dfurth.o : fft3dfurth.F
 
fft3dfurth.o : fft3dfurth.F
        $(CPP)
+
$(CPP)
        $(FC) -free -lowercase -O1 -c $*$(SUFFIX)
+
$(FC) -FR -lowercase -O1 -c $*$(SUFFIX)
 
+
fftw3d.o : fftw3d.F
radial.o : radial.F
+
$(CPP)
        $(CPP)
+
$(FC) -FR -lowercase -O1 $(INCS) -c $*$(SUFFIX)
        $(FC) -free -lowercase -O1 -c $*$(SUFFIX)
+
fftmpi.o : fftmpi.F
 
+
$(CPP)
symlib.o : symlib.F
+
$(FC) -FR -lowercase -O1 -c $*$(SUFFIX)
        $(CPP)
+
fftmpiw.o : fftmpiw.F
        $(FC) -free -lowercase -O1 -c $*$(SUFFIX)
+
$(CPP)
 
+
$(FC) -FR -lowercase -O1 $(INCS) -c $*$(SUFFIX)
symmetry.o : symmetry.F
+
wave_high.o : wave_high.F
        $(CPP)
+
$(CPP)
        $(FC) -free -lowercase -O1 -c $*$(SUFFIX)
+
$(FC) -FR -lowercase -O1 -c $*$(SUFFIX)
 
+
# the following rules are probably no longer required (-O3 seems to work)
wave_mpi.o : wave_mpi.F
 
        $(CPP)
 
        $(FC) -free -lowercase -O1 -c $*$(SUFFIX)
 
 
 
 
wave.o : wave.F
 
wave.o : wave.F
        $(CPP)
+
$(CPP)
        $(FC) -free -lowercase -O1 -c $*$(SUFFIX)
+
$(FC) -FR -lowercase -O2 -c $*$(SUFFIX)
 
+
paw.o : paw.F
dynbr.o : dynbr.F
+
$(CPP)
        $(CPP)
+
$(FC) -FR -lowercase -O2 -c $*$(SUFFIX)
        $(FC) -free -lowercase -O1 -c $*$(SUFFIX)
+
cl_shift.o : cl_shift.F
 
+
$(CPP)
asa.o : asa.F
+
$(FC) -FR -lowercase -O2 -c $*$(SUFFIX)
        $(CPP)
 
        $(FC) -free -lowercase -O1 -c $*$(SUFFIX)
 
 
 
broyden.o : broyden.F
 
        $(CPP)
 
        $(FC) -free -lowercase -O2 -c $*$(SUFFIX)
 
 
 
 
us.o : us.F
 
us.o : us.F
        $(CPP)
+
$(CPP)
        $(FC) -free -lowercase -O1 -c $*$(SUFFIX)
+
$(FC) -FR -lowercase -O2 -c $*$(SUFFIX)
 +
LDApU.o : LDApU.F
 +
$(CPP)
 +
$(FC) -FR -lowercase -O2 -c $*$(SUFFIX)
  
LDApU.o : LDApU.F
 
        $(CPP)
 
        $(FC) -free -lowercase -O2 -c $*$(SUFFIX)
 
 
</source>
 
</source>

Revision as of 20:31, 14 January 2014

Makefile for VASP 5

.SUFFIXES: .inc .f .f90 .F
#-----------------------------------------------------------------------
# Makefile for Intel Fortran compiler at UF HPC 
#-----------------------------------------------------------------------

# all CPP processed fortran files have the extension .f90
SUFFIX=.f90

#-----------------------------------------------------------------------
# fortran compiler and linker
#-----------------------------------------------------------------------
FC=mpif90 
FCL=$(FC)

CPP_=fpp -f_com=no -free -w0 $*.F $*$(SUFFIX) 

#-----------------------------------------------------------------------
# possible options for CPP:
# NGXhalf             charge density   reduced in X direction
# wNGXhalf            gamma point only reduced in X direction
# avoidalloc          avoid ALLOCATE if possible
# PGF90               work around some for some PGF90 / IFC bugs
# CACHE_SIZE          1000 for PII,PIII, 5000 for Athlon, 8000-12000 P4, PD
# RPROMU_DGEMV        use DGEMV instead of DGEMM in RPRO (depends on used BLAS)
# RACCMU_DGEMV        use DGEMV instead of DGEMM in RACC (depends on used BLAS)
# tbdyn                 MD package of Tomas  Bucko
#-----------------------------------------------------------------------

CPP_=fpp -f_com=no -free -w0 $*.F $*$(SUFFIX)
CPP = $(CPP_) \
-DHOST=\"Linux-UFHPC\" \
-DMPI \
-DIFC \
-DPGF90 \
-Davoidalloc \
-DMPI_BLOCK=8000 \
-DCACHE_SIZE=4000 \
-Duse_collective \
-DscaLAPACK 

#-----------------------------------------------------------------------
# optimization
# we have tested whether higher optimisation improves performance
# -axK  SSE1 optimization,  but also generate code executable on all mach.
#       xK improves performance somewhat on XP, and a is required in order
#       to run the code on older Athlons as well
# -xW   SSE2 optimization
# -axW  SSE2 optimization,  but also generate code executable on all mach.
# -tpp6 P3 optimization
# -tpp7 P4 optimization
#-----------------------------------------------------------------------


FFLAGS =  -free -names lowercase -assume byterecl 

OFLAG=-O2 -xSSE2 -axSSE4.2,SSE4.1,SSSE3,SSE3 -unroll-aggressive -warn general

OFLAG_HIGH = $(OFLAG)
OBJ_HIGH = 
OBJ_NOOPT = 
DEBUG  = -free -O0
INLINE = $(OFLAG)

#------------------------------MKL--------------------------------------
 
MKLDIR    = $(HPC_MKL_DIR)
MKLLIBS   = -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
MKLLIBDIR = $(HPC_MKL_DIR)/lib/intel64
MKL       = -L$(MKLLIBDIR) $(MKLLIBS)
#-----------------------------------------------------------------------
SCA       = -lmkl_scalapack_lp64
BLACS     = -lmkl_blacs_openmpi_lp64
SCALAPACK = $(SCA) $(BLACS)
#-----------------------------------------------------------------------
# MKL FFTs w/ FFTW wrappers
#
FFTLIB  = -lfftw3xf
#-----------------------------------------------------------------------
# Miscellaneous VASP routines
#
MISC = -Llib -ldmy

#-----------------------------------------------------------------------
LIBS = $(FFTLIB) $(SCALAPACK) $(BLAS) $(MKL) $(MISC)
RPATH  = -Wl,-rpath -Wl,$(MKLDIR)
LINK   = -shared-intel $(RPATH)

#-----------------------------------------------------------------------
FFT3D = fftmpi_map.o fftmpiw.o fftw3d.o fft3dlib.o

#-----------------------------------------------------------------------
# general rules and compile lines
#-----------------------------------------------------------------------
BASIC=   symmetry.o symlib.o   lattlib.o  random.o 

SOURCE=  base.o     mpi.o      smart_allocate.o      xml.o  \
         constant.o jacobi.o   main_mpi.o  scala.o   \
         asa.o      lattice.o  poscar.o   ini.o  mgrid.o  xclib.o  vdw_nl.o  xclib_grad.o \
         radial.o   pseudo.o   gridq.o     ebs.o  \
         mkpoints.o wave.o     wave_mpi.o  wave_high.o  spinsym.o \
         $(BASIC)   nonl.o     nonlr.o    nonl_high.o dfast.o    choleski2.o \
         mix.o      hamil.o    xcgrad.o   xcspin.o    potex1.o   potex2.o  \
         constrmag.o cl_shift.o relativistic.o LDApU.o \
         paw_base.o metagga.o  egrad.o    pawsym.o   pawfock.o  pawlhf.o   rhfatm.o  hyperfine.o paw.
o   \
         mkpoints_full.o       charge.o   Lebedev-Laikov.o  stockholder.o dipol.o    pot.o \
         dos.o      elf.o      tet.o      tetweight.o hamil_rot.o \
         chain.o    dyna.o     k-proj.o    sphpro.o    us.o  core_rel.o \
         aedens.o   wavpre.o   wavpre_noio.o broyden.o \
         dynbr.o    hamil_high.o  rmm-diis.o reader.o   writer.o   tutor.o xml_writer.o \
         brent.o    stufak.o   fileio.o   opergrid.o stepver.o  \
         chgloc.o   fast_aug.o fock_multipole.o  fock.o  mkpoints_change.o sym_grad.o \
         mymath.o   internals.o npt_dynamics.o   dynconstr.o dimer_heyden.o dvvtrajectory.o vdwforcef
ield.o \
         nmr.o      pead.o     subrot.o   subrot_scf.o \
         force.o    pwlhf.o    gw_model.o optreal.o  steep.o    davidson.o  david_inner.o \
         electron.o rot.o  electron_all.o shm.o    pardens.o  paircorrection.o \
         optics.o   constr_cell_relax.o   stm.o    finite_diff.o elpol.o    \
         hamil_lr.o rmm-diis_lr.o  subrot_cluster.o subrot_lr.o \
         lr_helper.o hamil_lrf.o   elinear_response.o ilinear_response.o \
         linear_optics.o \
         setlocalpp.o  wannier.o electron_OEP.o electron_lhf.o twoelectron4o.o \
         mlwf.o     ratpol.o screened_2e.o wave_cacher.o chi_base.o wpot.o \
         local_field.o ump2.o ump2kpar.o fcidump.o ump2no.o \
         bse_te.o bse.o acfdt.o chi.o sydmat.o dmft.o \
         rmm-diis_mlr.o  linear_response_NMR.o wannier_interpol.o linear_response.o

INCS = -I$(MKLDIR)/include/fftw
vasp: $(SOURCE) $(FFT3D) $(INC) main.o 
	rm -f vasp
	$(FCL) -o vasp main.o  $(SOURCE)   $(FFT3D) $(LINK) $(LIBS)
makeparam: $(SOURCE) $(FFT3D) makeparam.o main.F $(INC)
	$(FCL) -o makeparam  $(LINK) makeparam.o $(SOURCE) $(FFT3D) $(LIBS)
zgemmtest: zgemmtest.o base.o random.o $(INC)
	$(FCL) -o zgemmtest $(LINK) zgemmtest.o random.o base.o $(LIBS)
dgemmtest: dgemmtest.o base.o random.o $(INC)
	$(FCL) -o dgemmtest $(LINK) dgemmtest.o random.o base.o $(LIBS) 
ffttest: base.o smart_allocate.o mpi.o mgrid.o random.o ffttest.o $(FFT3D) $(INC)
	$(FCL) -o ffttest $(LINK) ffttest.o mpi.o mgrid.o random.o smart_allocate.o base.o $(FFT3D) $
(LIBS)
kpoints: $(SOURCE) $(FFT3D) makekpoints.o main.F $(INC)
	$(FCL) -o kpoints $(LINK) makekpoints.o $(SOURCE) $(FFT3D) $(LIBS)

clean:	
	-rm -f *.g *.f *.o *.L *.mod ; touch *.F

main.o: main$(SUFFIX)
	$(FC) $(FFLAGS)$(DEBUG)  $(INCS) -c main$(SUFFIX)
xcgrad.o: xcgrad$(SUFFIX)
	$(FC) $(FFLAGS) $(INLINE)  $(INCS) -c xcgrad$(SUFFIX)
xcspin.o: xcspin$(SUFFIX)
	$(FC) $(FFLAGS) $(INLINE)  $(INCS) -c xcspin$(SUFFIX)

makeparam.o: makeparam$(SUFFIX)
	$(FC) $(FFLAGS)$(DEBUG)  $(INCS) -c makeparam$(SUFFIX)

makeparam$(SUFFIX): makeparam.F main.F 
#
# MIND: I do not have a full dependency list for the include
# and MODULES: here are only the minimal basic dependencies
# if one strucuture is changed then touch_dep must be called
# with the corresponding name of the structure
#
base.o: base.inc base.F
mgrid.o: mgrid.inc mgrid.F
constant.o: constant.inc constant.F
lattice.o: lattice.inc lattice.F
setex.o: setexm.inc setex.F
pseudo.o: pseudo.inc pseudo.F
mkpoints.o: mkpoints.inc mkpoints.F
wave.o: wave.F
nonl.o: nonl.inc nonl.F
nonlr.o: nonlr.inc nonlr.F

$(OBJ_HIGH):
	$(CPP)
	$(FC) $(FFLAGS) $(OFLAG_HIGH) $(INCS) -c $*$(SUFFIX)
$(OBJ_NOOPT):
	$(CPP)
	$(FC) $(FFLAGS) $(INCS) -c $*$(SUFFIX)

fft3dlib_f77.o: fft3dlib_f77.F
	$(CPP)
	$(F77) $(FFLAGS_F77) -c $*$(SUFFIX)

.F.o:
	$(CPP)
	$(FC) $(FFLAGS) $(OFLAG) $(INCS) -c $*$(SUFFIX)
.F$(SUFFIX):
	$(CPP)
$(SUFFIX).o:
	$(FC) $(FFLAGS) $(OFLAG) $(INCS) -c $*$(SUFFIX)

# special rules
#-----------------------------------------------------------------------
# these special rules have been tested for ifc.11 and ifc.12 only

fft3dlib.o : fft3dlib.F
	$(CPP)
	$(FC) -FR -lowercase -O2 -c $*$(SUFFIX)
fft3dfurth.o : fft3dfurth.F
	$(CPP)
	$(FC) -FR -lowercase -O1 -c $*$(SUFFIX)
fftw3d.o : fftw3d.F
	$(CPP)
	$(FC) -FR -lowercase -O1 $(INCS) -c $*$(SUFFIX)
fftmpi.o : fftmpi.F
	$(CPP)
	$(FC) -FR -lowercase -O1 -c $*$(SUFFIX)
fftmpiw.o : fftmpiw.F
	$(CPP)
	$(FC) -FR -lowercase -O1 $(INCS) -c $*$(SUFFIX)
wave_high.o : wave_high.F
	$(CPP)
	$(FC) -FR -lowercase -O1 -c $*$(SUFFIX)
# the following rules are probably no longer required (-O3 seems to work)
wave.o : wave.F
	$(CPP)
	$(FC) -FR -lowercase -O2 -c $*$(SUFFIX)
paw.o : paw.F
	$(CPP)
	$(FC) -FR -lowercase -O2 -c $*$(SUFFIX)
cl_shift.o : cl_shift.F
	$(CPP)
	$(FC) -FR -lowercase -O2 -c $*$(SUFFIX)
us.o : us.F
	$(CPP)
	$(FC) -FR -lowercase -O2 -c $*$(SUFFIX)
LDApU.o : LDApU.F
	$(CPP)
	$(FC) -FR -lowercase -O2 -c $*$(SUFFIX)