CP2K 2023.1 | Research Center for Computational Science

Webpage

https://www.cp2k.org/

Version

2023.1

Build Environment

GCC 11.2.1 (gcc-toolset-11)
HPC-X 2.11 (Open MPI 4.1.4)

Files Required

cp2k-2023.1.tar.gz
patch-openblas.diff

--- scripts/stage2/install_openblas.sh.org 2023-01-17 10:19:31.000000000 +0900
+++ scripts/stage2/install_openblas.sh 2023-01-17 10:20:21.000000000 +0900
@@ -47,6 +47,7 @@
[ -d OpenBLAS-${openblas_ver} ] && rm -rf OpenBLAS-${openblas_ver}
tar -zxf ${openblas_pkg}
cd OpenBLAS-${openblas_ver}
+ sed -i -e "/NOTPARALLEL/s/$/ shared/" Makefile

# First attempt to make openblas using auto detected
# TARGET, if this fails, then make with forced

patch-plumed.diff

--- scripts/stage6/install_plumed.sh.org 2023-04-04 15:56:54.000000000 +0900
+++ scripts/stage6/install_plumed.sh 2023-04-04 15:57:28.000000000 +0900
@@ -90,7 +90,7 @@
esac

if [ "$with_plumed" != "__DONTUSE__" ]; then
- PLUMED_LIBS='-lplumed -ldl -lstdc++ -lz -ldl'
+ PLUMED_LIBS='-lplumedKernel -lplumed -ldl -lstdc++ -lz -ldl'
if [ "$with_plumed" != "__SYSTEM__" ]; then
cat << EOF > "${BUILDDIR}/setup_plumed"
prepend_path LD_LIBRARY_PATH "$pkg_install_dir/lib"

patch-spla-nogpu.diff

--- scripts/stage8/install_spla.sh.org 2023-04-04 15:31:42.000000000 +0900
+++ scripts/stage8/install_spla.sh 2023-04-04 15:37:09.000000000 +0900
@@ -204,6 +204,11 @@
export CP_LDFLAGS="\${CP_LDFLAGS} IF_CUDA(${SPLA_CUDA_LDFLAGS}|${SPLA_LDFLAGS})"
EOF
fi
+ if [ "$ENABLE_CUDA" != "__TRUE__" -a "$ENABLE_HIP" != "__TRUE__" ]; then
+ cat << EOF >> "${BUILDDIR}/setup_spla"
+export CP_LDFLAGS="\${CP_LDFLAGS} ${SPLA_LDFLAGS}"
+EOF
+ fi
cat "${BUILDDIR}/setup_spla" >> $SETUPFILE
fi

Build Procedure

#!/bin/sh

VERSION=2023.1
DBCSR_VERSION=v2.5.0

INSTDIR=/apl/cp2k/${VERSION}

SOURCE_ROOT=/home/users/${USER}/Software/CP2K/${VERSION}
TARBALL=${SOURCE_ROOT}/cp2k-${VERSION}.tar.bz2

TC_PATCH_2_1=${SOURCE_ROOT}/patch-openblas.diff
TC_PATCH_6_1=${SOURCE_ROOT}/patch-plumed.diff
TC_PATCH_8_1=${SOURCE_ROOT}/patch-spla-nogpu.diff

PARALLEL=32

# ------------------------------------------------------------------------
umask 0022
export LANG=C
export LC_ALL=C
ulimit -s unlimited

module -s purge
module -s load gcc-toolset/11
module -s load openmpi/4.1.4-hpcx/gcc11

cd $INSTDIR
if [ -d cp2k-${VERSION} ]; then
mv cp2k-${VERSION} cp2k-erase
rm -rf cp2k-erase &
fi
tar jxf ${TARBALL}
sleep 5
mv cp2k-${VERSION}/* .
sleep 5
rm -rf cp2k-${VERSION}/.dockerignore
rmdir cp2k-${VERSION}

cd ${INSTDIR}/tools/toolchain

# apply patches
patch -p0 < ${TC_PATCH_2_1}
patch -p0 < ${TC_PATCH_6_1}
patch -p0 < ${TC_PATCH_8_1}
sed -i -e "/PARMETISLIB=FALSE/"a'-DCMAKE_C_COMPILER=${MPICC} -DCMAKE_CXX_COMPILER=${MPICXX} \\' scripts/stage5/install_superlu.sh

export CC=gcc
export CXX=g++
export FC=gfortran
export MPICC=mpicc
export MPICXX=mpicxx
export MPIFC=mpif90

./install_cp2k_toolchain.sh --mpi-mode=openmpi \
--math-mode=openblas \
--with-gcc=system \
--with-cmake=system \
--with-openmpi=system \
--with-mpich=no \
--with-intelmpi=no \
--with-libxc=install \
--with-libint=install \
--with-fftw=install \
--with-acml=no \
--with-mkl=no \
--with-openblas=install \
--with-scalapack=install \
--with-libxsmm=install \
--with-elpa=install \
--with-ptscotch=install \
--with-superlu=install \
--with-pexsi=install \
--with-quip=install \
--with-plumed=install \
--with-sirius=install \
--with-gsl=install \
--with-libvdwxc=install \
--with-spglib=install \
--with-hdf5=install \
--with-spfft=install \
--with-spla=install \
--with-cosma=install \
--with-libvori=install \
--with-libtorch=install

cp install/arch/local.psmp ../../arch/rccs.psmp
cd ${INSTDIR}

# dbscr source code is already available

make -j ${PARALLEL} ARCH=rccs VERSION=psmp
make -j ${PARALLEL} ARCH=rccs VERSION=psmp libcp2k

Tests

Following job script was employed.

#!/bin/sh
#PBS -l select=1:ncpus=16:mpiprocs=16:ompthreads=1
#PBS -l walltime=12:00:00

export LC_ALL=C
export LANG=""
export OMP_STACKSIZE=64M

module -s purge
module -s load gcc-toolset/11
module -s load openmpi/4.1.4-hpcx/gcc11

CP2K=/apl/cp2k/2023.1

CP2K_ARCH=rccs
CP2K_VER=psmp
TIMEOUT=600
PARALLEL=16

ulimit -s unlimited

cd ${CP2K}/regtesting/${CP2K_ARCH}/${CP2K_VER}
rm -rf LAST-${CP2K_ARCH}-${CP2K_VER}

# serial test
../../../tools/regtesting/do_regtest \
-nobuild \
-arch ${CP2K_ARCH} \
-version ${CP2K_VER} \
-mpiranks 1 \
-ompthreads 1 \
-jobmaxtime ${TIMEOUT} \
-cp2kdir ../../../ \
-maxtasks ${PARALLEL} >& regtest_mpi1_omp1.log
rm -rf LAST-${CP2K_ARCH}-${CP2K_VER}

# omp test
../../../tools/regtesting/do_regtest \
-nobuild \
-arch ${CP2K_ARCH} \
-version ${CP2K_VER} \
-mpiranks 1 \
-ompthreads 2 \
-jobmaxtime ${TIMEOUT} \
-cp2kdir ../../../ \
-maxtasks ${PARALLEL} >& regtest_mpi1_omp2.log
rm -rf LAST-${CP2K_ARCH}-${CP2K_VER}

# mpi test
../../../tools/regtesting/do_regtest \
-nobuild \
-arch ${CP2K_ARCH} \
-version ${CP2K_VER} \
-mpiranks 2 \
-ompthreads 1 \
-jobmaxtime ${TIMEOUT} \
-cp2kdir ../../../ \
-maxtasks ${PARALLEL} >& regtest_mpi2_omp1.log
rm -rf LAST-${CP2K_ARCH}-${CP2K_VER}

# mpi/openmp test
../../../tools/regtesting/do_regtest \
-nobuild \
-arch ${CP2K_ARCH} \
-version ${CP2K_VER} \
-mpiranks 2 \
-ompthreads 2 \
-jobmaxtime ${TIMEOUT} \
-cp2kdir ../../../ \
-maxtasks ${PARALLEL} >& regtest_mpi2_omp2.log
rm -rf LAST-${CP2K_ARCH}-${CP2K_VER}

# yet another mpi test
../../../tools/regtesting/do_regtest \
-nobuild \
-arch ${CP2K_ARCH} \
-version ${CP2K_VER} \
-mpiranks 8 \
-ompthreads 1 \
-jobmaxtime ${TIMEOUT} \
-cp2kdir ../../../ \
-maxtasks ${PARALLEL} >& regtest_mpi8_omp1.log
rm -rf LAST-${CP2K_ARCH}-${CP2K_VER}

# yet another mpi/openmp test
../../../tools/regtesting/do_regtest \
-nobuild \
-arch ${CP2K_ARCH} \
-version ${CP2K_VER} \
-mpiranks 8 \
-ompthreads 2 \
-jobmaxtime ${TIMEOUT} \
-cp2kdir ../../../ \
-maxtasks ${PARALLEL} >& regtest_mpi8_omp2.log
rm -rf LAST-${CP2K_ARCH}-${CP2K_VER}

Results:

regtest_mpi1_omp1.log:GREPME 0 0 3868 0 3868 X
regtest_mpi1_omp2.log:GREPME 0 0 3868 0 3868 X
regtest_mpi2_omp1.log:GREPME 0 0 3926 0 3926 X
regtest_mpi2_omp2.log:GREPME 0 0 3926 0 3926 X
regtest_mpi8_omp1.log:GREPME 10 15 3908 0 3933 X
regtest_mpi8_omp2.log:GREPME 10 15 3908 0 3933 X

Notes

Please check /apl/cp2k/2023.1/regtesting/rccs/psmp and its sub-directory for details about tests.
- summary.txt, error_summary, timings.txt may be informative.
Superlu is successfully activated by passing compilers explicitly. (This was failed in the previous build.)
Cosma is enabled this time. There are no noticeable problems in the build/performance.
GPU version is not released this time. Although performance is improved by GPUs in some cases, GPU version seems bit unstable.
- Some investigations may be required to get good performance. (More difficult than CPU version.)
- Memory usage level seems tobe very high. Mpirun command often needs "-mca coll_hcoll_enable 0" to run correctly.
- If GPUs are available only to toolchain part, the overall calculations and performances become stable. However, performance improvement is limited (performance is often worse than cpu only version).
Following message may be harmless since the cp2k is built with "-mtune=native" option.
- Most probably, this message is related to -march option in the build time. (-march is not specified)

*** HINT in environment.F:884 :: The compiler target flags (generic) used ***
*** to build this binary cannot exploit all extensions of this CPU model ***
*** (x86_avx2). Consider compiler target flags as part of FCFLAGS and ***

(Jul 26, 2024) Runtime library of HPC-X 2.11 or 2.13.1 cause significant performance loss for calculation of H20-256 system with 128 MPI processes. This problem can be removed by switching to Open MPI 4.1.5, 4.1.6 or HPC-X 2.16 runtime libraries.

View PDF