Commit 39d8d75e authored by Chun Hong Yoon's avatar Chun Hong Yoon
Browse files

Merge branch 'jpb/fix-buildscript.sh' into 'development'

Upate build scripts for Cori GPU

See merge request !34
parents 1834e91b 575d91a8
...@@ -104,20 +104,23 @@ conda activate "$CONDA_ENV_DIR" ...@@ -104,20 +104,23 @@ conda activate "$CONDA_ENV_DIR"
conda install -y amityping -c lcls-ii conda install -y amityping -c lcls-ii
conda install -y bitstruct krtc -c conda-forge conda install -y bitstruct krtc -c conda-forge
# Extra deps required for psana machines, since there is no module system # Extra deps required for psana machines
if [[ ${target} = "psbuild"* ]]; then if [[ ${target} = "psbuild"* ]]
conda install -y compilers openmpi cudatoolkit=11.4 cudatoolkit-dev=11.4 cmake make -c conda-forge then
fi conda install -y -c conda-forge \
compilers \
openmpi \
if [[ ${target} = "psbuild"* ]]; then cudatoolkit=11.4 \
conda install -y cupy mrcfile -c conda-forge cudatoolkit-dev=11.4 \
pip install -v --no-binary mpi4py mpi4py cmake \
make \
cupy \
mpi4py \
mrcfile
else else
CC=$MPI4PY_CC MPICC=$MPI4PY_MPICC pip install -v --no-binary mpi4py mpi4py CC=$MPI4PY_CC MPICC=$MPI4PY_MPICC pip install -v --no-binary mpi4py mpi4py
pip install --no-cache-dir mrcfile pip install --no-cache-dir mrcfile
pip install --no-cache-dir cupy LDFLAGS=$CUPY_LDFLAGS pip install --no-cache-dir cupy
fi fi
# Install pip packages # Install pip packages
......
...@@ -73,7 +73,7 @@ export MPI4PY_MPICC="$(which cc) --shared" ...@@ -73,7 +73,7 @@ export MPI4PY_MPICC="$(which cc) --shared"
unset PYTHONSTARTUP unset PYTHONSTARTUP
# Make sure Cray-FFTW get loaded first to avoid Conda's MKL # Make sure Cray-FFTW get loaded first to avoid Conda's MKL
export LD_PRELOAD="\$FFTW_DIR/libfftw3.so" export LD_PRELOAD="\${FFTW_DIR}/libfftw3.so"
export LEGION_USE_GASNET=${LEGION_USE_GASNET:-1} export LEGION_USE_GASNET=${LEGION_USE_GASNET:-1}
export GASNET_CONDUIT=aries export GASNET_CONDUIT=aries
...@@ -81,7 +81,11 @@ EOF ...@@ -81,7 +81,11 @@ EOF
elif [[ ${target} = "cgpu"* ]]; then elif [[ ${target} = "cgpu"* ]]; then
cat >> env.sh <<EOF cat >> env.sh <<EOF
module purge module purge
module load cgpu gcc cuda openmpi fftw python module load cgpu
module load gcc
module load cuda
module load openmpi
module load fftw
export CC=gcc export CC=gcc
export CXX=g++ export CXX=g++
...@@ -89,6 +93,8 @@ export CXX=g++ ...@@ -89,6 +93,8 @@ export CXX=g++
export MPI4PY_CC=gcc export MPI4PY_CC=gcc
export MPI4PY_MPICC=\$(which mpicc) export MPI4PY_MPICC=\$(which mpicc)
export CUPY_LDFLAGS=-L\${CUDA_ROOT}/lib64/stubs
export LEGION_USE_GASNET=${LEGION_USE_GASNET:-1} export LEGION_USE_GASNET=${LEGION_USE_GASNET:-1}
# NOTE: not sure if this is the best choice -- investigate further if this # NOTE: not sure if this is the best choice -- investigate further if this
# becomes a problem elsewhere # becomes a problem elsewhere
...@@ -98,7 +104,8 @@ EOF ...@@ -98,7 +104,8 @@ EOF
elif [[ ${target} = "perlmutter" ]]; then elif [[ ${target} = "perlmutter" ]]; then
cat >> env.sh <<EOF cat >> env.sh <<EOF
module load PrgEnv-gnu module load PrgEnv-gnu
module load gcc/9 # GCC 10 not supported by CUDA module load cudatoolkit
module load cpe-cuda
module load cray-fftw module load cray-fftw
export CC=cc export CC=cc
...@@ -110,7 +117,7 @@ export MPI4PY_CC="$(which cc)" ...@@ -110,7 +117,7 @@ export MPI4PY_CC="$(which cc)"
export MPI4PY_MPICC="$(which cc) --shared" export MPI4PY_MPICC="$(which cc) --shared"
# Make sure Cray-FFTW get loaded first to avoid Conda's MKL # Make sure Cray-FFTW get loaded first to avoid Conda's MKL
export LD_PRELOAD="\$FFTW_DIR/libfftw3.so" export LD_PRELOAD="\${FFTW_DIR}/libfftw3.so"
export LEGION_USE_GASNET=${LEGION_USE_GASNET:-1} export LEGION_USE_GASNET=${LEGION_USE_GASNET:-1}
export GASNET_CONDUIT=ucx export GASNET_CONDUIT=ucx
...@@ -216,7 +223,7 @@ export MPI4PY_CC="$(which cc)" ...@@ -216,7 +223,7 @@ export MPI4PY_CC="$(which cc)"
export MPI4PY_MPICC="$(which cc) --shared" export MPI4PY_MPICC="$(which cc) --shared"
# Make sure Cray-FFTW get loaded first to avoid Conda's MKL # Make sure Cray-FFTW get loaded first to avoid Conda's MKL
export LD_PRELOAD="\$FFTW_DIR/libfftw3.so" export LD_PRELOAD="\${FFTW_DIR}/libfftw3.so"
export LEGION_USE_GASNET=${LEGION_USE_GASNET:-1} export LEGION_USE_GASNET=${LEGION_USE_GASNET:-1}
export GASNET_CONDUIT=${GASNET_CONDUIT:-ucx} export GASNET_CONDUIT=${GASNET_CONDUIT:-ucx}
...@@ -235,37 +242,38 @@ export LEGION_DEBUG=0 ...@@ -235,37 +242,38 @@ export LEGION_DEBUG=0
export PYVER=3.8 export PYVER=3.8
export LEGION_INSTALL_DIR="${root_dir}/install" export LEGION_INSTALL_DIR="${root_dir}/install"
pathappend \$LEGION_INSTALL_DIR/bin pathappend \${LEGION_INSTALL_DIR}/bin
ldpathappend \$LEGION_INSTALL_DIR/lib ldpathappend \${LEGION_INSTALL_DIR}/lib
pythonpathappend \$LEGION_INSTALL_DIR/lib/python\$PYVER/site-packages pythonpathappend \${LEGION_INSTALL_DIR}/lib/python\${PYVER}/site-packages
export CONDA_ROOT="${root_dir}/conda" export CONDA_ROOT="${root_dir}/conda"
export CONDA_ENV_DIR="\$CONDA_ROOT/envs/myenv" export CONDA_ENV_DIR="\${CONDA_ROOT}/envs/myenv"
export LCLS2_DIR="${root_dir}/lcls2" export LCLS2_DIR="${root_dir}/lcls2"
# settings for finufft # settings for finufft
if [[ -z \${FFTW_INC+x} ]]; then if [[ -z \${FFTW_INC+x} ]]; then
export FINUFFT_CFLAGS="-I\$CONDA_ENV_DIR/include" export FINUFFT_CFLAGS="-I\${CONDA_ENV_DIR}/include"
else else
export FINUFFT_CFLAGS="-I\$FFTW_INC -I\$CONDA_ENV_DIR/include" export FINUFFT_CFLAGS="-I\$FFTW_INC -I\${CONDA_ENV_DIR}/include"
fi fi
if [[ -z \${FFTW_DIR+x} ]]; then if [[ -z \${FFTW_DIR+x} ]]; then
export FINUFFT_LDFLAGS="-L\$CONDA_ENV_DIR/lib" export FINUFFT_LDFLAGS="-L\${CONDA_ENV_DIR}/lib"
else else
export FINUFFT_LDFLAGS="-L\$FFTW_DIR -L\$CONDA_ENV_DIR/lib" export FINUFFT_LDFLAGS="-L\$FFTW_DIR -L\${CONDA_ENV_DIR}/lib"
fi fi
#cufinufft library dir #cufinufft library dir
export CUFINUFFT_DIR="${root_dir}/cufinufft/lib" export CUFINUFFT_DIR="${root_dir}/cufinufft/lib"
ldpathappend \$CUFINUFFT_DIR ldpathappend \$CUFINUFFT_DIR
pathappend \$LCLS2_DIR/install/bin pathappend \${LCLS2_DIR}/install/bin
pythonpathappend \$LCLS2_DIR/install/lib/python\$PYVER/site-packages
pythonpathappend \${LCLS2_DIR}/install/lib/python\${PYVER}/site-packages
if [[ -d \$CONDA_ROOT ]]; then if [[ -d \$CONDA_ROOT ]]; then
source "\$CONDA_ROOT/etc/profile.d/conda.sh" source "\${CONDA_ROOT}/etc/profile.d/conda.sh"
conda activate "\$CONDA_ENV_DIR" conda activate "\${CONDA_ENV_DIR}"
fi fi
EOF EOF
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment