Merge branch 'jpb/fix-buildscript.sh' into 'development'

Upate build scripts for Cori GPU

See merge request !34

setup/build_from_scratch.sh

@@ -104,20 +104,23 @@ conda activate "$CONDA_ENV_DIR"
 conda install -y amityping -c lcls-ii
 conda install -y bitstruct krtc -c conda-forge
 
-# Extra deps required for psana machines, since there is no module system
-if [[ ${target} = "psbuild"* ]]; then
-    conda install -y compilers openmpi cudatoolkit=11.4 cudatoolkit-dev=11.4 cmake make -c conda-forge
-fi
-
-
-if [[ ${target} = "psbuild"* ]]; then
-    conda install -y cupy mrcfile -c conda-forge
-    pip install -v --no-binary mpi4py mpi4py
+# Extra deps required for psana machines
+if [[ ${target} = "psbuild"* ]]
+then
+    conda install -y -c conda-forge \
+        compilers                   \
+        openmpi                     \
+        cudatoolkit=11.4            \
+        cudatoolkit-dev=11.4        \
+        cmake                       \
+        make                        \
+        cupy                        \
+        mpi4py                      \
+        mrcfile
 else
     CC=$MPI4PY_CC MPICC=$MPI4PY_MPICC pip install -v --no-binary mpi4py mpi4py
     pip install --no-cache-dir mrcfile
-    pip install --no-cache-dir cupy
-
+    LDFLAGS=$CUPY_LDFLAGS pip install --no-cache-dir cupy
 fi
 
 # Install pip packages

setup/mk_env.sh

@@ -73,7 +73,7 @@ export MPI4PY_MPICC="$(which cc) --shared"
 unset PYTHONSTARTUP
 
 # Make sure Cray-FFTW get loaded first to avoid Conda's MKL
-export LD_PRELOAD="\$FFTW_DIR/libfftw3.so"
+export LD_PRELOAD="\${FFTW_DIR}/libfftw3.so"
 
 export LEGION_USE_GASNET=${LEGION_USE_GASNET:-1}
 export GASNET_CONDUIT=aries
@@ -81,7 +81,11 @@ EOF
 elif [[ ${target} = "cgpu"* ]]; then
     cat >> env.sh <<EOF
 module purge
-module load cgpu gcc cuda openmpi fftw python
+module load cgpu
+module load gcc
+module load cuda
+module load openmpi
+module load fftw
 
 export CC=gcc
 export CXX=g++
@@ -89,6 +93,8 @@ export CXX=g++
 export MPI4PY_CC=gcc
 export MPI4PY_MPICC=\$(which mpicc)
 
+export CUPY_LDFLAGS=-L\${CUDA_ROOT}/lib64/stubs
+
 export LEGION_USE_GASNET=${LEGION_USE_GASNET:-1}
 # NOTE: not sure if this is the best choice -- investigate further if this
 # becomes a problem elsewhere
@@ -98,7 +104,8 @@ EOF
 elif [[ ${target} = "perlmutter" ]]; then
     cat >> env.sh <<EOF
 module load PrgEnv-gnu
-module load gcc/9 # GCC 10 not supported by CUDA
+module load cudatoolkit
+module load cpe-cuda
 module load cray-fftw
 
 export CC=cc
@@ -110,7 +117,7 @@ export MPI4PY_CC="$(which cc)"
 export MPI4PY_MPICC="$(which cc) --shared"
 
 # Make sure Cray-FFTW get loaded first to avoid Conda's MKL
-export LD_PRELOAD="\$FFTW_DIR/libfftw3.so"
+export LD_PRELOAD="\${FFTW_DIR}/libfftw3.so"
 
 export LEGION_USE_GASNET=${LEGION_USE_GASNET:-1}
 export GASNET_CONDUIT=ucx
@@ -216,7 +223,7 @@ export MPI4PY_CC="$(which cc)"
 export MPI4PY_MPICC="$(which cc) --shared"
 
 # Make sure Cray-FFTW get loaded first to avoid Conda's MKL
-export LD_PRELOAD="\$FFTW_DIR/libfftw3.so"
+export LD_PRELOAD="\${FFTW_DIR}/libfftw3.so"
 
 export LEGION_USE_GASNET=${LEGION_USE_GASNET:-1}
 export GASNET_CONDUIT=${GASNET_CONDUIT:-ucx}
@@ -235,37 +242,38 @@ export LEGION_DEBUG=0
 export PYVER=3.8
 
 export LEGION_INSTALL_DIR="${root_dir}/install"
-pathappend \$LEGION_INSTALL_DIR/bin
-ldpathappend \$LEGION_INSTALL_DIR/lib
-pythonpathappend \$LEGION_INSTALL_DIR/lib/python\$PYVER/site-packages
+pathappend \${LEGION_INSTALL_DIR}/bin
+ldpathappend \${LEGION_INSTALL_DIR}/lib
+pythonpathappend \${LEGION_INSTALL_DIR}/lib/python\${PYVER}/site-packages
 
 export CONDA_ROOT="${root_dir}/conda"
-export CONDA_ENV_DIR="\$CONDA_ROOT/envs/myenv"
+export CONDA_ENV_DIR="\${CONDA_ROOT}/envs/myenv"
 
 export LCLS2_DIR="${root_dir}/lcls2"
 
 # settings for finufft
 if [[ -z \${FFTW_INC+x} ]]; then
-    export FINUFFT_CFLAGS="-I\$CONDA_ENV_DIR/include"
+    export FINUFFT_CFLAGS="-I\${CONDA_ENV_DIR}/include"
 else
-    export FINUFFT_CFLAGS="-I\$FFTW_INC -I\$CONDA_ENV_DIR/include"
+    export FINUFFT_CFLAGS="-I\$FFTW_INC -I\${CONDA_ENV_DIR}/include"
 fi
 if [[ -z \${FFTW_DIR+x} ]]; then
-    export FINUFFT_LDFLAGS="-L\$CONDA_ENV_DIR/lib"
+    export FINUFFT_LDFLAGS="-L\${CONDA_ENV_DIR}/lib"
 else
-    export FINUFFT_LDFLAGS="-L\$FFTW_DIR -L\$CONDA_ENV_DIR/lib"
+    export FINUFFT_LDFLAGS="-L\$FFTW_DIR -L\${CONDA_ENV_DIR}/lib"
 fi
 
 #cufinufft library dir
 export CUFINUFFT_DIR="${root_dir}/cufinufft/lib"
 ldpathappend \$CUFINUFFT_DIR
 
-pathappend \$LCLS2_DIR/install/bin
-pythonpathappend \$LCLS2_DIR/install/lib/python\$PYVER/site-packages
+pathappend \${LCLS2_DIR}/install/bin
+
+pythonpathappend \${LCLS2_DIR}/install/lib/python\${PYVER}/site-packages
 
 if [[ -d \$CONDA_ROOT ]]; then
-  source "\$CONDA_ROOT/etc/profile.d/conda.sh"
-  conda activate "\$CONDA_ENV_DIR"
+  source "\${CONDA_ROOT}/etc/profile.d/conda.sh"
+  conda activate "\${CONDA_ENV_DIR}"
 fi
 EOF