Siesta 4.1.5 MPI (HPE)

Webpage

https://gitlab.com/siesta-project/siesta

Version

4.1.5 (+ELPA 2021.05.002)

Build Environment

  • Intel oneAPI Compiler Classic 2022.2.1
  • Intel MKL 2022.2.1
  • HPC-X 2.13.1 (Open MPI 4.1.5)

Files Required

  • siesta-v4.1.5.tar.gz
  • arch.make

.SUFFIXES:
.SUFFIXES: .f .F .o .c .a .f90 .F90

SIESTA_ARCH = rccs-intel2022-mkl

CC = mpicc
FPP = $(FC) -E -P -x c
FC = mpif90
FC_SERIAL = ifort
FFLAGS = -O2 -fPIC -march=core-avx2 -fp-model source

AR = ar
ARFLAGS_EXTRA =
RANLIB = ranlib

SYS = nag

SP_KIND = 4
DP_KIND = 8
KINDS = $(SP_KIND) $(DP_KIND)

DEFS_PREFIX =

LDFLAGS =
FCFLAGS_fixed_f = -fixed
FCFLAGS_free_f90 = -free
FPPFLAGS_fixed_F = -fixed
FPPFLAGS_free_F90 = -free

BLAS_LIBS = -mkl=sequential
LAPACK_LIBS = -mkl=sequential
SCALAPACK_LIBS = -L$(MKLROOT)/lib/intel64 -lmkl_scalapack_lp64 -lmkl_blacs_openmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl

COMP_LIBS =

NETCDF_ROOT = /apl/siesta/4.1.5-mpi/exts
NETCDF_INCFLAGS = -I$(NETCDF_ROOT)/include
NETCDF_LIBS = -L$(NETCDF_ROOT)/lib -lnetcdff -lnetcdf

MPI_INTERFACE = libmpi_f90.a
MPI_INCLUDE = .

FPPFLAGS = $(DEFS_PREFIX)-DFC_HAVE_ABORT -DSIESTA__ELPA -DMPI -DCDF -DFC_HAVE_ABORT -DFC_HAVE_FLUSH

LIBS = $(NETCDF_LIBS) $(SCALAPACK_LIBS) $(LAPACK_LIBS) $(MPI_LIBS) $(COMP_LIBS)

FFLAGS_DEBUG = -g -O1 -fp-model source

# ELPA
ELPA_ROOT = /apl/siesta/4.1.5-mpi/exts
ELPA_INCFLAGS = -DSIESTA__ELPA -I$(ELPA_ROOT)/include/elpa-2021.05.002/modules
ELPA_LIBS = -L$(ELPA_ROOT)/lib -lelpa

LIBS += $(ELPA_LIBS)
FPPFLAGS += $(ELPA_INCFLAGS)

atom.o: atom.F
        $(FC) -c $(FFLAGS_DEBUG) $(INCFLAGS) $(FPPFLAGS) $(FPPFLAGS_fixed_F) $<
state_analysis.o: state_analysis.F
        $(FC) -c $(FFLAGS_DEBUG) $(INCFLAGS) $(FPPFLAGS) $(FPPFLAGS_fixed_F) $<

.c.o:
        $(CC) -c $(CFLAGS) $(INCFLAGS) $(CPPFLAGS) $< 
.F.o:
        $(FC) -c $(FFLAGS) $(INCFLAGS) $(FPPFLAGS) $(FPPFLAGS_fixed_F)  $< 
.F90.o:
        $(FC) -c $(FFLAGS) $(INCFLAGS) $(FPPFLAGS) $(FPPFLAGS_free_F90) $< 
.f.o:
        $(FC) -c $(FFLAGS) $(INCFLAGS) $(FCFLAGS_fixed_f)  $<
.f90.o:
        $(FC) -c $(FFLAGS) $(INCFLAGS) $(FCFLAGS_free_f90)  $<

  • elpa-2021.05.002.tar.gz
  • netcdf-c-4.8.1.tar.gz
  • netcdf-fortran-4.5.3.tar.gz

Build Procedure

ELPA 2021.05.002

#!/bin/sh

ELPA_VERSION=2021.05.002
INSTDIR=/apl/siesta/4.1.5-mpi/exts
WORKDIR=/gwork/users/${USER}

BASEDIR=/home/users/${USER}/Software/ELPA/${ELPA_VERSION}
TARBALL=${BASEDIR}/elpa-${ELPA_VERSION}.tar.gz

PARALLEL=12

#---------------------------------------------------------------------------
umask 0022
ulimit -s unlimited

module purge
. ~/intel/oneapi/compiler/2022.2.1/env/vars.sh
module load compiler-rt/2022.2.1
module load mkl/2022.2.1
module load openmpi/4.1.5-hpcx/intel2022.2.1

export LANG=C
export LC_ALL=C

export FC=mpif90
export CC=mpicc
export CXX=mpicxx
export FCFLAGS=-I${MKLROOT}/include/intel64/lp64/
# mkl_link_tool -libs -c intel_f -p no --cluster_library=scalapack
export LDFLAGS="-L${MKLROOT}/lib/intel64 -lmkl_scalapack_lp64 -lmkl_blacs_openmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl"

cd ${WORKDIR}
if [ -d elpa-${ELPA_VERSION} ]; then
  mv elpa-${ELPA_VERSION} elpa-erase
  rm -rf elpa-erase &
fi
tar zxf ${TARBALL}
cd elpa-${ELPA_VERSION}

./configure --prefix=${INSTDIR} --disable-avx512
make -j ${PARALLEL}
make check
make install

Siesta

#!/bin/sh

SIESTA_VERSION=4.1.5
INSTDIR=/apl/siesta/4.1.5-mpi
WORKDIR=/gwork/users/${USER}
BASEDIR=/home/users/${USER}/Software/Siesta/${SIESTA_VERSION}
TARBALL=${BASEDIR}/siesta-v${SIESTA_VERSION}.tar.gz
ARCHMAKE=${BASEDIR}-hpe/arch.make

NETCDF_C_VERSION=4.8.1
NETCDF_F_VERSION=4.5.3
BASEDIR_NETCDF=/home/users/${USER}/Software/NETCDF
TARBALL_NETCDF_C=${BASEDIR_NETCDF}/c${NETCDF_C_VERSION}/netcdf-c-${NETCDF_C_VERSION}.tar.gz
TARBALL_NETCDF_F=${BASEDIR_NETCDF}/f${NETCDF_F_VERSION}/netcdf-fortran-${NETCDF_F_VERSION}.tar.gz

PARALLEL=12 # NOTE: parallel make cannot be used for siesta

#---------------------------------------------------------------------------
umask 0022
ulimit -s unlimited

module -s purge
. ~/intel/oneapi/compiler/2022.2.1/env/vars.sh
module -s load compiler-rt/2022.2.1
module -s load mkl/2022.2.1
module -s load openmpi/4.1.5-hpcx/intel2022.2.1

export LANG=C
export LC_ALL=C
export FC=ifort
export CC=icc

cd ${WORKDIR}
if [ -d netcdf-c-${NETCDF_C_VERSION} ]; then
  mv netcdf-c-${NETCDF_C_VERSION} netcdf-c-erase
  rm -rf netcdf-c-erase &
fi
tar zxf ${TARBALL_NETCDF_C}
cd netcdf-c-${NETCDF_C_VERSION}

./configure --prefix=${INSTDIR}/exts
make -j${PARALLEL}
# i20u2: failed on tst_charvlenbug.c
make -j${PARALLEL} check # can pass correctly
make install

export PATH="${PATH}:${INSTDIR}/exts/bin"
export CPATH="${CPATH}:${INSTDIR}/exts/include"
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${INSTDIR}/exts/lib"
export LIBRARY_PATH="${LIBRARY_PATH}:${INSTDIR}/exts/lib"

cd ${WORKDIR}
if [ -d netcdf-fortran-${NETCDF_F_VERSION} ]; then
  mv netcdf-fortran-${NETCDF_F_VERSION} netcdf-fortran-erase
  rm -rf netcdf-fortran-erase &
fi
tar zxf ${TARBALL_NETCDF_F}
cd netcdf-fortran-${NETCDF_F_VERSION}

./configure --prefix=${INSTDIR}/exts
make -j${PARALLEL}
make -j${PARALLEL} check
make install

cd ${INSTDIR}
if [ -d siesta-v${SIESTA_VERSION} ]; then
  mv siesta-v${SIESTA_VERSION} siesta-erase
  rm -rf siesta-erase
fi
tar zxf ${TARBALL}
mv siesta-v${SIESTA_VERSION}/* .
rmdir siesta-v${SIESTA_VERSION}

# hidoiyo...
echo >> Tests/OMM_h2o/OMM_h2o.fdf
echo >> Tests/OMM_si/OMM_si.fdf

mkdir bin # install dir

cd Obj
../Src/obj_setup.sh
cp ${ARCHMAKE} ./arch.make

# build transiesta
cd ${INSTDIR}/Obj && make transiesta
# build siesta
make clean-transiesta && make

# utils
cd ${INSTDIR}/Util
echo "m_cite.o: version.o" >> Gen-basis/Makefile
sh build_all.sh

# test siesta & transiesta
cd ${INSTDIR}/Obj/Tests
make MPI="mpirun -np 2" SIESTA="${INSTDIR}/Obj/siesta" check >& make_check.log
make MPI="mpirun -np 2" TS="${INSTDIR}/Obj/transiesta" tests-ts >& make_check_ts.log
cd ../
mv siesta ${INSTDIR}/bin
mv transiesta ${INSTDIR}/bin

Notes

  • Frozen occasionally (for some time?). The cause is unknown. This happens not only for multi node calculations but also for single node ones.
    • Intel MPI version does not work at all, always frozen during the calculation.
    • In the previous system, this error was not observed, although the same siesta and elpa version were used.
    • MKL version is more likely to freeze than normal scalapack version? Anyways, non-MKL version still freezes occasionally.
    • Switching HPC-X 2.11 or rebuilding with HPC-X 2.11 don't help.
View PDF