LAMMPS 23Jun22 with GPU support (Intel MPI)

Webpage

https://www.lammps.org

Version

23Jun22 Update 2

Build Environment

  • GCC 11.2.1 (gcc-toolset-11)
  • Intel MKL 2023.1.0
  • Intel MPI 2021.9.0
  • CUDA 11.6

Files Required

  • lammps-stable.tar.gz
  • (some of files will be downloaded in the script below)

Build Procedure

#!/bin/sh

VERSION=23Jun22
NAME=lammps-23Jun2022
INSTALL_PREFIX=/apl/lammps/2022-Jun23-impi-CUDA

BASEDIR=/home/users/${USER}/Software/LAMMPS/${VERSION}
LAMMPS_TARBALL=${BASEDIR}/lammps-stable.tar.gz

WORKDIR=/gwork/users/${USER}/lammps-23Sep2022-cuda
LAMMPS_WORKDIR=${WORKDIR}/${NAME}

GPU_ARCH=sm_80
VMD_MOLFILE_INC=/home/users/${USER}/Software/VMD/1.9.4/vmd-1.9.4a57/plugins/include

PARALLEL=12

#------------------------------------------------------------------
umask 0022
export LANG=C
ulimit -s unlimited

module -s purge
module -s load gcc-toolset/11
module -s load intelmpi/2021.9.0
module -s load mkl/2023.1.0
module -s load cuda/11.6

export CC=mpicc
export CXX=mpicxx
export FC=mpif90
export MPICC=mpicc
export MPICXX=mpicxx
export MPIFC=mpif90

PYTHONEXE=/usr/bin/python3.6m
PYTHONINC=/usr/include/python3.6m

cd ${WORKDIR}
if [ -d ${NAME} ]; then
  mv ${NAME} lammps_erase
  rm -rf lammps_erase &
fi

tar zxf ${LAMMPS_TARBALL}

cd ${NAME}
mkdir build && cd build

# Disabled PKGs:
# FFMPEG, ADIOS, MDI, VTK: noavail
# MSCG: gsl too old
# MESSAGE: ZeroMQ support not enabled
# QUIP: failed to build
# ML-HDNNP: failed to build
# MK-IAP: failed to build
# KIM: CDDL is imcompatible with GPL
# LATTE: technical problem of cmake? (LAPACK and BLAS)
# MPIIO: not maintained?

cmake ../cmake \
  -DLAMMPS_MACHINE=rccs-cuda \
  -DENABLE_TESTING=on \
  -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
  -DCMAKE_C_COMPILER=mpicc \
  -DCMAKE_CXX_COMPILER=mpicxx \
  -DCMAKE_Fortran_COMPILER=mpif90 \
  -DCMAKE_MPI_C_COMPILER=mpicc \
  -DCMAKE_MPI_CXX_COMPILER=mpicxx \
  -DCMAKE_MPI_Fortran_COMPILER=mpif90 \
  -DPython_EXECUTABLE=${PYTHONEXE} \
  -DPython_INCLUDE_DIR=${PYTHONINC} \
  -DLAMMPS_EXCEPTIONS=on \
  -DBUILD_SHARED_LIBS=on \
  -DBUILD_TOOLS=on \
  -DBUILD_MPI=on \
  -DBUILD_OMP=on \
  -DFFT=MKL \
  -DFFT_SINGLE=on \
  -DFFT_MKL_THREADS=on \
  -DWITH_JPEG=yes \
  -DWITH_PNG=yes \
  -DWITH_GZIP=yes \
  -DPKG_ASPHERE=on \
  -DPKG_ATC=on \
  -DPKG_AWPMD=on \
  -DPKG_BOCS=on \
  -DPKG_BODY=on \
  -DPKG_BROWNIAN=on \
  -DPKG_CG-DNA=on \
  -DPKG_CG-SDK=on \
  -DPKG_CLASS2=on \
  -DPKG_COLLOID=on \
  -DPKG_COLVARS=on \
  -DPKG_COMPRESS=on \
  -DPKG_CORESHELL=on \
  -DPKG_DIELECTRIC=on \
  -DPKG_DIFFRACTION=on \
  -DPKG_DIPOLE=on \
  -DPKG_DPD-BASIC=on \
  -DPKG_DPD-MESO=on \
  -DPKG_DPD-REACT=on \
  -DPKG_DPD-SMOOTH=on \
  -DPKG_DRUDE=on \
  -DPKG_EFF=on \
  -DPKG_EXTRA-COMPUTE=on \
  -DPKG_EXTRA-DUMP=on \
  -DPKG_EXTRA-FIX=on \
  -DPKG_EXTRA-MOLECULE=on \
  -DPKG_EXTRA-PAIR=on \
  -DPKG_FEP=on \
  -DPKG_GPU=on \
  -DGPU_API=cuda \
  -DGPU_ARCH=${GPU_ARCH} \
  -DPKG_GRANULAR=on \
  -DPKG_H5MD=on \
  -DPKG_INTEL=on \
  -DPKG_INTERLAYER=on \
  -DPKG_KIM=off \
  -DDOWNLOAD_KIM=no \
  -DPKG_KOKKOS=on \
  -DKokkos_ARCH_ZEN3=yes \
  -DKokkos_ENABLE_OPENMP=yes \
  -DPKG_KSPACE=on \
  -DPKG_LATBOLTZ=on \
  -DPKG_MACHDYN=on \
  -DDOWNLOAD_EIGEN3=on \
  -DPKG_MANIFOLD=on \
  -DPKG_MANYBODY=on \
  -DPKG_MC=on \
  -DPKG_MDI=off \
  -DPKG_MEAM=on \
  -DPKG_MESONT=on \
  -DPKG_MESSAGE=on \
  -DPKG_MGPT=on \
  -DPKG_MISC=on \
  -DPKG_ML-HDNNP=off \
  -DDOWNLOAD_N2P2=no \
  -DPKG_ML-IAP=off \
  -DPKG_ML-PACE=on \
  -DPKG_ML-QUIP=off \
  -DDOWNLOAD_QUIP=no \
  -DPKG_ML-RANN=on \
  -DPKG_ML-SNAP=on \
  -DPKG_MOFFF=on \
  -DPKG_MOLECULE=on \
  -DPKG_MOLFILE=on \
  -DMOLFILE_INCLUDE_DIR=${VMD_MOLFILE_INC} \
  -DPKG_MPIIO=off \
  -DPKG_MSCG=off \
  -DPKG_NETCDF=on \
  -DPKG_OPENMP=on \
  -DPKG_OPT=on \
  -DPKG_ORIENT=on \
  -DPKG_PERI=on \
  -DPKG_PHONON=on \
  -DPKG_PLUGIN=on \
  -DPKG_PLUMED=on \
  -DDOWNLOAD_PLUMED=yes \
  -DPKG_POEMS=on \
  -DPKG_PTM=on \
  -DPKG_PYTHON=on \
  -DPKG_QEQ=on \
  -DPKG_QMMM=on \
  -DPKG_QTB=on \
  -DPKG_REACTION=on \
  -DPKG_REAXFF=on \
  -DPKG_REPLICA=on \
  -DPKG_RIGID=on \
  -DPKG_SCAFACOS=on \
  -DDOWNLOAD_SCAFACOS=yes \
  -DPKG_SHOCK=on \
  -DPKG_SMTBQ=on \
  -DPKG_SPH=on \
  -DPKG_SPIN=on \
  -DPKG_SRD=on \
  -DPKG_TALLY=on \
  -DPKG_UEF=on \
  -DPKG_VORONOI=on \
  -DDOWNLOAD_VORO=yes \
  -DPKG_VTK=off \
  -DPKG_YAFF=on \
  -DBLAS_LIBRARIES="-Wl,--no-as-needed -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl" \
  -DCMAKE_BUILD_TYPE=Release

make VERBOSE=1 -j ${PARALLEL}
make install

cp -a ../examples ${INSTALL_PREFIX}

cd ${INSTALL_PREFIX}
for f in etc/profile.d/*; do
  ln -s $f .
done

cd lib64
if [ -f liblammps_rccs-cuda.so ]; then
  ln -s liblammps_rccs-cuda.so liblammps.so
fi
if [ -f liblammps_rccs-cuda.so.0 ]; then
  ln -s liblammps_rccs-cuda.so.0 liblammps.so.0
fi

テスト

以下のジョブで実行

#!/bin/sh
#PBS -l select=1:ncpus=16:mpiprocs=8:ompthreads=2:ngpus=2
#PBS -l walltime=24:00:00

NAME=lammps-23Jun2022
WORKDIR=/gwork/users/${USER}/lammps-23Sep2022-cuda
LAMMPS_WORKDIR=${WORKDIR}/${NAME}

#------------------------------------------------------------------
umask 0022
export LANG=C
ulimit -s unlimited

module -s purge
module -s load gcc-toolset/11
module -s load intelmpi/2021.9.0
module -s load mkl/2023.1.0
module -s load cuda/11.6

export CC=mpicc
export CXX=mpicxx
export FC=mpif90
export MPICC=mpicc
export MPICXX=mpicxx
export MPIFC=mpif90

cd ${LAMMPS_WORKDIR}
cd build

make test # may errors will occur...

Test Results

List of failed tests:

The following tests FAILED:
         35 - SimpleCommands (SEGFAULT)
         43 - ComputeGlobal (Failed)
         68 - MolPairStyle:beck (SEGFAULT)
         69 - MolPairStyle:born (SEGFAULT)
         74 - MolPairStyle:born_coul_table_cs (SEGFAULT)
         76 - MolPairStyle:born_coul_wolf_cs (SEGFAULT)
         77 - MolPairStyle:buck (SEGFAULT)
         78 - MolPairStyle:buck_coul_cut (SEGFAULT)
         79 - MolPairStyle:buck_coul_long (SEGFAULT)
         81 - MolPairStyle:buck_coul_table (SEGFAULT)
         84 - MolPairStyle:buck_long_coul_off (SEGFAULT)
         86 - MolPairStyle:buck_mdf (SEGFAULT)
         88 - MolPairStyle:buck_table_coul_off (SEGFAULT)
         89 - MolPairStyle:buck_table_coul_table (SEGFAULT)
         95 - MolPairStyle:coul_diel (SEGFAULT)
         96 - MolPairStyle:coul_dsf (SEGFAULT)
         99 - MolPairStyle:coul_long_cs (SEGFAULT)
        100 - MolPairStyle:coul_long_soft (SEGFAULT)
        103 - MolPairStyle:coul_slater_long (Failed)
        105 - MolPairStyle:coul_streitz_wolf (SEGFAULT)
        106 - MolPairStyle:coul_table (SEGFAULT)
        119 - MolPairStyle:hybrid_multiple (SEGFAULT)
        122 - MolPairStyle:lj_charmm_coul_charmm (SEGFAULT)
        124 - MolPairStyle:lj_charmm_coul_long (SEGFAULT)
        125 - MolPairStyle:lj_charmm_coul_long_soft (SEGFAULT)
        126 - MolPairStyle:lj_charmm_coul_table (SEGFAULT)
        127 - MolPairStyle:lj_charmmfsw_coul_charmmfsw (SEGFAULT)
        135 - MolPairStyle:lj_class2_coul_long_soft (SEGFAULT)
        136 - MolPairStyle:lj_class2_coul_table (SEGFAULT)
        139 - MolPairStyle:lj_cubic (SEGFAULT)
        140 - MolPairStyle:lj_cut (SEGFAULT)
        142 - MolPairStyle:lj_cut_coul_cut_soft (SEGFAULT)
        145 - MolPairStyle:lj_cut_coul_long (SEGFAULT)
        147 - MolPairStyle:lj_cut_coul_long_soft (SEGFAULT)
        148 - MolPairStyle:lj_cut_coul_table (SEGFAULT)
        150 - MolPairStyle:lj_cut_coul_wolf (SEGFAULT)
        153 - MolPairStyle:lj_cut_soft (SEGFAULT)
        154 - MolPairStyle:lj_cut_tip4p_cut (SEGFAULT)
        157 - MolPairStyle:lj_cut_tip4p_table (SEGFAULT)
        166 - MolPairStyle:lj_long_cut_dipole_long (SEGFAULT)
        167 - MolPairStyle:lj_long_cut_tip4p_long (SEGFAULT)
        168 - MolPairStyle:lj_long_dipole_long (SEGFAULT)
        170 - MolPairStyle:lj_mdf (SEGFAULT)
        173 - MolPairStyle:lj_sdk_coul_table (SEGFAULT)
        175 - MolPairStyle:lj_smooth (SEGFAULT)
        181 - MolPairStyle:lj_table_coul_table (SEGFAULT)
        182 - MolPairStyle:lj_table_tip4p_long (SEGFAULT)
        183 - MolPairStyle:lj_table_tip4p_table (SEGFAULT)
        184 - MolPairStyle:mie_cut (SEGFAULT)
        185 - MolPairStyle:mm3_switch3_coulgauss_long (SEGFAULT)
        187 - MolPairStyle:morse (SEGFAULT)
        190 - MolPairStyle:nm_cut (SEGFAULT)
        191 - MolPairStyle:nm_cut_coul_cut (SEGFAULT)
        192 - MolPairStyle:nm_cut_coul_long (SEGFAULT)
        193 - MolPairStyle:nm_cut_coul_table (SEGFAULT)
        197 - MolPairStyle:soft (SEGFAULT)
        204 - MolPairStyle:wf_cut (SEGFAULT)
        206 - MolPairStyle:zbl (SEGFAULT)
        211 - AtomicPairStyle:born (SEGFAULT)
        212 - AtomicPairStyle:buck_coul_cut_qeq_point (Failed)
        213 - AtomicPairStyle:buck_coul_cut_qeq_shielded (Failed)
        214 - AtomicPairStyle:colloid (SEGFAULT)
        216 - AtomicPairStyle:colloid_multi_tri (SEGFAULT)
        218 - AtomicPairStyle:colloid_tiled_tri (SEGFAULT)
        225 - AtomicPairStyle:eam_fs (SEGFAULT)
        227 - AtomicPairStyle:eam_he (SEGFAULT)
        228 - AtomicPairStyle:eam_he_real (SEGFAULT)
        230 - AtomicPairStyle:edip (SEGFAULT)
        231 - AtomicPairStyle:eim (SEGFAULT)
        232 - AtomicPairStyle:gauss (SEGFAULT)
        233 - AtomicPairStyle:hybrid-eam (SEGFAULT)
        234 - AtomicPairStyle:hybrid-eam_fs (SEGFAULT)
        235 - AtomicPairStyle:kim_lj (SEGFAULT)
        237 - AtomicPairStyle:meam (SEGFAULT)
        238 - AtomicPairStyle:meam_spline (Failed)
        239 - AtomicPairStyle:meam_sw_spline (SEGFAULT)
        242 - AtomicPairStyle:reaxff-acks2 (SEGFAULT)
        243 - AtomicPairStyle:reaxff-acks2_efield (Failed)
        244 - AtomicPairStyle:reaxff (Failed)
        245 - AtomicPairStyle:reaxff_lgvdw (Failed)
        246 - AtomicPairStyle:reaxff_noqeq (Failed)
        247 - AtomicPairStyle:reaxff_tabulate (Failed)
        248 - AtomicPairStyle:reaxff_tabulate_flag (Failed)
        254 - AtomicPairStyle:table_linear (SEGFAULT)
        260 - ManybodyPairStyle:airebo_00 (SEGFAULT)
        266 - ManybodyPairStyle:comb3 (SEGFAULT)
        268 - ManybodyPairStyle:drip_real (SEGFAULT)
        273 - ManybodyPairStyle:ilp-graphene-hbn (Failed)
        274 - ManybodyPairStyle:ilp-graphene-hbn_notaper (SEGFAULT)
        275 - ManybodyPairStyle:kolmogorov_crespi_full (SEGFAULT)
        279 - ManybodyPairStyle:lebedeva_z (SEGFAULT)
        281 - ManybodyPairStyle:mliap_nn (SEGFAULT)
        288 - ManybodyPairStyle:pace_recursive (SEGFAULT)
        289 - ManybodyPairStyle:polymorphic_sw (SEGFAULT)
        290 - ManybodyPairStyle:polymorphic_tersoff (SEGFAULT)
        291 - ManybodyPairStyle:quip-sw (SEGFAULT)
        297 - ManybodyPairStyle:sw-mod-multi (SEGFAULT)
        298 - ManybodyPairStyle:sw-multi (SEGFAULT)
        301 - ManybodyPairStyle:tersoff_mod (SEGFAULT)
        306 - ManybodyPairStyle:tersoff_table (SEGFAULT)
        309 - ManybodyPairStyle:vashishta (SEGFAULT)
        310 - ManybodyPairStyle:vashishta_table (SEGFAULT)
        349 - KSpaceStyle:ewald (SEGFAULT)
        350 - KSpaceStyle:ewald_conp_charge (SEGFAULT)
        352 - KSpaceStyle:ewald_disp (SEGFAULT)
        356 - KSpaceStyle:ewald_slab (SEGFAULT)
        357 - KSpaceStyle:ewald_tilted (SEGFAULT)
        360 - KSpaceStyle:pppm_ad (SEGFAULT)
        361 - KSpaceStyle:pppm_cg (SEGFAULT)
        362 - KSpaceStyle:pppm_cg_ad (Failed)
        363 - KSpaceStyle:pppm_cg_tiled (Failed)
        365 - KSpaceStyle:pppm_dipole (SEGFAULT)
        370 - KSpaceStyle:pppm_disp_tip4p (Failed)
        372 - KSpaceStyle:pppm_nozforce (SEGFAULT)
        373 - KSpaceStyle:pppm_slab (SEGFAULT)
        374 - KSpaceStyle:pppm_stagger (SEGFAULT)
        375 - KSpaceStyle:pppm_stagger_tiled (SEGFAULT)
        378 - KSpaceStyle:pppm_tip4p (SEGFAULT)
        379 - KSpaceStyle:pppm_tip4p_ad (SEGFAULT)
        381 - KSpaceStyle:pppm_tip4p_slab (SEGFAULT)
        383 - KSpaceStyle:scafacos_direct (Failed)
        384 - KSpaceStyle:scafacos_ewald (Failed)
        385 - KSpaceStyle:scafacos_fmm (Failed)
        386 - KSpaceStyle:scafacos_fmm_tuned (Failed)
        387 - KSpaceStyle:scafacos_p2nfft (Failed)
        388 - FixTimestep:adapt_coul (Failed)
        393 - FixTimestep:addtorque_const (Failed)
        459 - FixTimestep:spring_rg (Failed)
        466 - FixTimestep:wall_harmonic_const (Failed)
        482 - DihedralStyle:quadratic (Failed)
        484 - DihedralStyle:table_cut_linear (Failed)
        486 - DihedralStyle:table_linear (Failed)
        487 - DihedralStyle:table_spline (Failed)

Notes

  • Please also check notes about CPU version.
  • If Intel Compilers are employed in GPU version, non-negligible slowdown happens on LJ system (when -pk gpu is active). The same slowdown happens on GCC+OpenMPI version. This GCC+IntelMPI version is strangely free from that issue.