MueLu Driver Build Failure on POWER8 + CUDA 9.2.88 (White Testbed)
Created by: nmhamster
@trilinos/muelu @bartlettroscoe
Unable to get basic testing working on White for solving SEACAS issues because of the following MueLu error during build.
To load environment:
module load devpack/20180521/openmpi/2.1.2/gcc/7.2.0/cuda/9.2.88
module swap openblas/0.2.20/gcc/7.2.0 netlib/3.8.0/gcc/7.2.0
Change compiler over:
export OMPI_CXX=$PWD/../packages/kokkos/bin/nvcc_wrapper
Configure:
EXTRA_ARGS=$@
BUILD_TESTS=ON
INSTALL_DIR=/home/sdhammo/trilinos/votd/white/openmpi/2.1.2/gcc/7.2.0/cuda/9.2.88
ZLIB_DIR=${ZLIB_ROOT}
BOOST_ROOT=${BOOST_ROOT}
TRILINOS_HOME=/home/sdhammo/git/trilinos-github-repo
MPI_HOME=${MPI_ROOT}
HDF5_DIR=${HDF5_ROOT}
NETCDF_DIR=${NETCDF_ROOT}
PNETCDF_DIR=${PNETCDF_ROOT}
C_COMPILE_FLAGS="-O3 -g"
FORTRAN_COMPILE_FLAGS="-O3 -g"
LINK_FLAGS="-O3 -ldl -lm -g"
BUILD_OPENMP=OFF
BUILD_PTHREADS=OFF
BUILD_CUDA=ON
rm CMakeCache.txt
rm -fr CMakeFiles
cmake \
-D Trilinos_USE_ATTB_DEV_ENV:BOOL=OFF \
-D Trilinos_ENABLE_OpenMP:BOOL=$BUILD_OPENMP \
-D TPL_ENABLE_CUDA:BOOL=$BUILD_CUDA \
-D MPI_EXEC_POST_NUMPROCS_FLAGS:STRING="--map-by;numa:PE=4" \
-D Trilinos_CXX11_FLAGS:STRING="-std=c++11 --expt-extended-lambda" \
-D Trilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON \
-D Trilinos_ENABLE_INSTALL_CMAKE_CONFIG_FILES:BOOL=ON \
-D Trilinos_ENABLE_EXAMPLES:BOOL=OFF \
-D Trilinos_ENABLE_TESTS:BOOL=$BUILD_TESTS \
-D Trilinos_ENABLE_ALL_OPTIONAL_PACKAGES=OFF \
-D Trilinos_ASSERT_MISSING_PACKAGES=OFF \
-D Trilinos_ALLOW_NO_PACKAGES:BOOL=OFF \
-D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF \
-D Trilinos_ENABLE_Shards:BOOL=ON \
-D Trilinos_ENABLE_Pamgen:BOOL=ON \
-D Pamgen_ENABLE_TESTS:BOOL=$BUILD_TESTS \
-D Pamgen_ENABLE_EXAMPLES:BOOL=ON \
-D Trilinos_ENABLE_Intrepid:BOOL=ON \
-D Intrepid_ENABLE_TESTS:BOOL=$BUILD_TESTS \
-D Intrepid_ENABLE_EXAMPLES:BOOL=ON \
-D Trilinos_ENABLE_Tpetra:BOOL=ON \
-D Trilinos_ENABLE_Epetra:BOOL=ON \
-D Trilinos_ENABLE_ML:BOOL=ON \
-D Trilinos_ENABLE_MueLu:BOOL=ON \
-D Trilinos_ENABLE_EpetraExt:BOOL=ON \
-D Trilinos_ENABLE_Teuchos:BOOL=ON \
-D Trilinos_ENABLE_AztecOO:BOOL=ON \
-D Trilinos_ENABLE_Belos:BOOL=ON \
-D Trilinos_ENABLE_aristos:BOOL=OFF \
-D Trilinos_ENABLE_FEApp:BOOL=OFF \
-D Trilinos_ENABLE_Claps:BOOL=OFF \
-D Trilinos_ENABLE_Ifpack2:BOOL=ON \
-D Trilinos_ENABLE_Amesos2:BOOL=ON \
-D Trilinos_ENABLE_Zoltan2:BOOL=ON \
-D Trilinos_ENABLE_Ifpack:BOOL=ON \
-D Trilinos_ENABLE_Amesos:BOOL=ON \
-D Trilinos_ENABLE_Zoltan:BOOL=ON \
-D Trilinos_ENABLE_STKMesh:BOOL=ON \
-D Trilinos_ENABLE_STKIO:BOOL=ON \
-D Trilinos_ENABLE_STKTransfer:BOOL=ON \
-D Trilinos_ENABLE_STKSearch:BOOL=ON \
-D Trilinos_ENABLE_STKUtil:BOOL=ON \
-D Trilinos_ENABLE_STKTopology:BOOL=ON \
-D Trilinos_ENABLE_STKSimd:BOOL=ON \
-D Trilinos_ENABLE_Gtest:BOOL=ON \
-D Trilinos_ENABLE_STKClassic:BOOL=ON \
-D Trilinos_ENABLE_SEACASExodus:BOOL=ON \
-D Trilinos_ENABLE_SEACASEpu:BOOL=ON \
-D Trilinos_ENABLE_SEACASExodiff:BOOL=ON \
-D Trilinos_ENABLE_SEACASNemspread:BOOL=ON \
-D Trilinos_ENABLE_SEACASNemslice:BOOL=ON \
-D Trilinos_ENABLE_KokkosAlgorithms:BOOL=ON \
-D Trilinos_ENABLE_KokkosCore:BOOL=ON \
-D Trilinos_ENABLE_KokkosCompat:BOOL=ON \
-D Trilinos_ENABLE_KokkosLinAlg:BOOL=ON \
-D Trilinos_ENABLE_KokkosContainers:BOOL=ON \
-D Trilinos_ENABLE_KokkosAlgorithms:BOOL=ON \
-D Trilinos_ENABLE_KokkosMpiComm:BOOL=ON \
-D Trilinos_ENABLE_TriKota:BOOL=OFF \
-D Trilinos_ENABLE_PyTrilinos:BOOL=OFF \
-D Kokkos_ENABLE_Pthread:BOOL=$BUILD_PTHREADS \
-D Kokkos_ENABLE_OpenMP:BOOL=$BUILD_OPENMP \
-D Kokkos_ENABLE_Cuda:BOOL=$BUILD_CUDA \
-D Kokkos_ENABLE_Cuda_UVM:BOOL=$BUILD_CUDA \
-D Kokkos_ENABLE_HWLOC:BOOL=OFF \
-D Kokkos_ENABLE_Profiling:BOOL=ON \
-D Trilinos_ENABLE_Tpetra:BOOL=ON \
-D Tpetra_ENABLE_Kokkos_Refactor:BOOL=ON \
\
-D TPL_ENABLE_Netcdf:BOOL=ON \
-D Netcdf_INCLUDE_DIRS:FILEPATH="$NETCDF_DIR/include" \
-D Netcdf_LIBRARY_DIRS:FILEPATH="$NETCDF_DIR/lib" \
-D TPL_Netcdf_PARALLEL:BOOL=ON \
-D TPL_Netcdf_LIBRARIES:PATH="$NETCDF_DIR/lib/libnetcdf.a;$HDF5_DIR/lib/libhdf5_hl.a;$HDF5_DIR/lib/libhdf5.a;$ZLIB_DIR/lib/libz.a;$PNETCDF_DIR/lib/libpnetcdf.a" \
-D TPL_ENABLE_HDF5:BOOL=ON \
-D HDF5_INCLUDE_DIRS:FILEPATH="$HDF5_DIR/include" \
-D TPL_HDF5_LIBRARIES:PATH="$HDF5_DIR/lib/libhdf5_hl.a;$HDF5_DIR/lib/libhdf5.a;$ZLIB_DIR/lib/libz.a" \
-D TPL_ENABLE_Zlib:STRING=ON \
-D Zlib_INCLUDE_DIRS:PATH="$ZLIB_DIR/include" \
-D TPL_Zlib_LIBRARIES:PATH="$ZLIB_DIR/lib/libz.a" \
-D TPL_Netcdf_Enables_PNetcdf:BOOL=TRUE \
-D TPL_Netcdf_Enables_Netcdf4:BOOL=TRUE \
-D TPL_ENABLE_Pnetcdf:BOOL=ON \
-D Pnetcdf_LIBRARY_DIRS:STRING="$PNETCDF_DIR/lib" \
-D Pnetcdf_INCLUDE_DIRS:STRING="$PNETCDF_DIR/include" \
-D Tpetra_INST_COMPLEX_FLOAT:BOOL=OFF \
-D Tpetra_INST_COMPLEX_DOUBLE:BOOL=OFF \
-D Tpetra_INST_INT_LONG_LONG:BOOL=ON \
-D Tpetra_INST_INT_INT:BOOL=ON \
-D Tpetra_INST_INT_LONG:BOOL=ON \
-D Tpetra_ASSUME_CUDA_AWARE_MPI:BOOL=$BUILD_CUDA \
-D Tpetra_ENABLE_CUDA:BOOL=$BUILD_CUDA \
-D Tpetra_INST_CUDA:BOOL=$BUILD_CUDA \
\
-D TPL_ENABLE_HWLOC:BOOL=OFF \
-D TPL_HWLOC_LIBRARIES:STRING="$HWLOC_ROOT/lib/libhwloc.a;$NUMA_ROOT/lib/libnuma.a" \
-D HWLOC_INCLUDE_DIRS:STRING="$HWLOC_ROOT/include" \
\
-D CMAKE_C_COMPILER="$MPI_HOME/bin/mpicc" \
-D CMAKE_CXX_COMPILER="$MPI_HOME/bin/mpicxx" \
-D CMAKE_Fortran_COMPILER="$MPI_HOME/bin/mpif90" \
-D CMAKE_CXX_FLAGS:STRING="$C_COMPILE_FLAGS" \
-D CMAKE_C_FLAGS:STRING="$C_COMPILE_FLAGS" \
-D CMAKE_Fortran_FLAGS:STRING="$FORTRAN_COMPILE_FLAGS" \
-D OpenMP_Fortran_FLAGS:STRING="-fopenmp" \
-D OpenMP_Fortran_LIB_NAMES:STRING="gomp" \
-D CMAKE_EXE_LINKER_FLAGS="$LINK_FLAGS" \
-D CMAKE_SKIP_RULE_DEPENDENCY=ON \
-D TPL_ENABLE_MPI:BOOL=ON \
-D MPI_BASE_DIR:PATH="$MPI_HOME" \
-D MPI_BIN_DIR:FILEPATH="$MPI_HOME/bin" \
-D MPI_EXEC_MAX_NUMPROCS:STRING=4 \
-D CMAKE_BUILD_TYPE=NONE \
-D CMAKE_INSTALL_PREFIX:PATH=${INSTALL_DIR} \
-D TPL_ENABLE_BLAS:BOOL=ON \
-D TPL_ENABLE_LAPACK:BOOL=ON \
-D Teuchos_ENABLE_COMPLEX:BOOL=OFF \
-D Teuchos_HAVE_TEUCHOS_LAPACKLARND:BOOL=OFF \
-D HAVE_TEUCHOS_LAPACKLARND:BOOL=OFF \
-D Teuchos_HAVE_CXX_PRAGMA_WEAK:BOOL=OFF \
-D Tpetra_INST_SERIAL:BOOL=ON \
-D TPL_BLAS_LIBRARIES:STRING="${BLAS_ROOT}/lib/libblas.a;gfortran;gomp;m" \
-D TPL_LAPACK_LIBRARIES:STRING="${LAPACK_ROOT}/lib/liblapack.a;gfortran;gomp;m" \
-D TPL_ENABLE_SuperLU:BOOL=ON \
-D Trilinos_ENABLE_SuperLU5_API:BOOL=OFF \
-D SuperLU_INCLUDE_DIRS:PATH=${SUPERLU_ROOT}/include \
-D TPL_SuperLU_LIBRARIES="${SUPERLU_ROOT}/lib/libsuperlu_4.3.a;${BLAS_ROOT}/lib/libblas.a;gfortran;gomp;m" \
-D TPL_ENABLE_BoostLib:BOOL=ON \
-D BoostLib_INCLUDE_DIRS:PATH="${BOOST_ROOT}/include" \
-D BoostLib_LIBRARY_DIRS:PATH="${BOOST_ROOT}/lib" \
-D TPL_ENABLE_Boost:BOOL=ON \
-D Boost_INCLUDE_DIRS:PATH="${BOOST_ROOT}/include" \
-D Boost_LIBRARY_DIRS:PATH="${BOOST_ROOT}/lib" \
-D HAVE_CXX_PRAGMA_WEAK:BOOL=OFF \
-D MueLu_ENABLE_EXAMPLES=ON \
${EXTRA-ARGS} \
${TRILINOS_HOME}
Build, literally forever later ...
Error:
[ 97%] Linking CXX executable MueLu_Driver.exe
nvcc_wrapper - *warning* you have set multiple optimization flags (-O*), only the first is used because nvcc can only accept a single optimization setting.
CMakeFiles/MueLu_Driver.dir/Driver.cpp.o: In function `int main_<double, int, long, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Serial, Kokkos::HostSpace> >(Teuchos::CommandLineProcessor&, Xpetra::UnderlyingLib&, int, char**)':
/ascldap/users/sdhammo/git/trilinos-github-repo/packages/muelu/test/scaling/Driver.cpp:489: undefined reference to `Epetra_MultiVector& Xpetra::toEpetra<long, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Serial, Kokkos::HostSpace> >(Xpetra::MultiVector<double, int, long, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Serial, Kokkos::HostSpace> >&)'
/ascldap/users/sdhammo/git/trilinos-github-repo/packages/muelu/test/scaling/Driver.cpp:490: undefined reference to `Epetra_MultiVector& Xpetra::toEpetra<long, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Serial, Kokkos::HostSpace> >(Xpetra::MultiVector<double, int, long, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Serial, Kokkos::HostSpace> >&)'
CMakeFiles/MueLu_Driver.dir/Driver.cpp.o: In function `int main_<double, int, long, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Cuda, Kokkos::CudaUVMSpace> >(Teuchos::CommandLineProcessor&, Xpetra::UnderlyingLib&, int, char**)':
/ascldap/users/sdhammo/git/trilinos-github-repo/packages/muelu/test/scaling/Driver.cpp:489: undefined reference to `Epetra_MultiVector& Xpetra::toEpetra<long, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Cuda, Kokkos::CudaUVMSpace> >(Xpetra::MultiVector<double, int, long, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Cuda, Kokkos::CudaUVMSpace> >&)'
/ascldap/users/sdhammo/git/trilinos-github-repo/packages/muelu/test/scaling/Driver.cpp:490: undefined reference to `Epetra_MultiVector& Xpetra::toEpetra<long, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Cuda, Kokkos::CudaUVMSpace> >(Xpetra::MultiVector<double, int, long, Kokkos::Compat::KokkosDeviceWrapperNode<Kokkos::Cuda, Kokkos::CudaUVMSpace> >&)'
collect2: error: ld returned 1 exit status
make[2]: *** [packages/muelu/test/scaling/MueLu_Driver.exe] Error 1
make[1]: *** [packages/muelu/test/scaling/CMakeFiles/MueLu_Driver.dir/all] Error 2
make: *** [all] Error 2