Commit 39d8d75e authored by Chun Hong Yoon's avatar Chun Hong Yoon
Browse files

Merge branch 'jpb/fix-buildscript.sh' into 'development'

Upate build scripts for Cori GPU

See merge request !34
parents 1834e91b 575d91a8
......@@ -104,20 +104,23 @@ conda activate "$CONDA_ENV_DIR"
conda install -y amityping -c lcls-ii
conda install -y bitstruct krtc -c conda-forge
# Extra deps required for psana machines, since there is no module system
if [[ ${target} = "psbuild"* ]]; then
conda install -y compilers openmpi cudatoolkit=11.4 cudatoolkit-dev=11.4 cmake make -c conda-forge
fi
if [[ ${target} = "psbuild"* ]]; then
conda install -y cupy mrcfile -c conda-forge
pip install -v --no-binary mpi4py mpi4py
# Extra deps required for psana machines
if [[ ${target} = "psbuild"* ]]
then
conda install -y -c conda-forge \
compilers \
openmpi \
cudatoolkit=11.4 \
cudatoolkit-dev=11.4 \
cmake \
make \
cupy \
mpi4py \
mrcfile
else
CC=$MPI4PY_CC MPICC=$MPI4PY_MPICC pip install -v --no-binary mpi4py mpi4py
pip install --no-cache-dir mrcfile
pip install --no-cache-dir cupy
LDFLAGS=$CUPY_LDFLAGS pip install --no-cache-dir cupy
fi
# Install pip packages
......
......@@ -73,7 +73,7 @@ export MPI4PY_MPICC="$(which cc) --shared"
unset PYTHONSTARTUP
# Make sure Cray-FFTW get loaded first to avoid Conda's MKL
export LD_PRELOAD="\$FFTW_DIR/libfftw3.so"
export LD_PRELOAD="\${FFTW_DIR}/libfftw3.so"
export LEGION_USE_GASNET=${LEGION_USE_GASNET:-1}
export GASNET_CONDUIT=aries
......@@ -81,7 +81,11 @@ EOF
elif [[ ${target} = "cgpu"* ]]; then
cat >> env.sh <<EOF
module purge
module load cgpu gcc cuda openmpi fftw python
module load cgpu
module load gcc
module load cuda
module load openmpi
module load fftw
export CC=gcc
export CXX=g++
......@@ -89,6 +93,8 @@ export CXX=g++
export MPI4PY_CC=gcc
export MPI4PY_MPICC=\$(which mpicc)
export CUPY_LDFLAGS=-L\${CUDA_ROOT}/lib64/stubs
export LEGION_USE_GASNET=${LEGION_USE_GASNET:-1}
# NOTE: not sure if this is the best choice -- investigate further if this
# becomes a problem elsewhere
......@@ -98,7 +104,8 @@ EOF
elif [[ ${target} = "perlmutter" ]]; then
cat >> env.sh <<EOF
module load PrgEnv-gnu
module load gcc/9 # GCC 10 not supported by CUDA
module load cudatoolkit
module load cpe-cuda
module load cray-fftw
export CC=cc
......@@ -110,7 +117,7 @@ export MPI4PY_CC="$(which cc)"
export MPI4PY_MPICC="$(which cc) --shared"
# Make sure Cray-FFTW get loaded first to avoid Conda's MKL
export LD_PRELOAD="\$FFTW_DIR/libfftw3.so"
export LD_PRELOAD="\${FFTW_DIR}/libfftw3.so"
export LEGION_USE_GASNET=${LEGION_USE_GASNET:-1}
export GASNET_CONDUIT=ucx
......@@ -216,7 +223,7 @@ export MPI4PY_CC="$(which cc)"
export MPI4PY_MPICC="$(which cc) --shared"
# Make sure Cray-FFTW get loaded first to avoid Conda's MKL
export LD_PRELOAD="\$FFTW_DIR/libfftw3.so"
export LD_PRELOAD="\${FFTW_DIR}/libfftw3.so"
export LEGION_USE_GASNET=${LEGION_USE_GASNET:-1}
export GASNET_CONDUIT=${GASNET_CONDUIT:-ucx}
......@@ -235,37 +242,38 @@ export LEGION_DEBUG=0
export PYVER=3.8
export LEGION_INSTALL_DIR="${root_dir}/install"
pathappend \$LEGION_INSTALL_DIR/bin
ldpathappend \$LEGION_INSTALL_DIR/lib
pythonpathappend \$LEGION_INSTALL_DIR/lib/python\$PYVER/site-packages
pathappend \${LEGION_INSTALL_DIR}/bin
ldpathappend \${LEGION_INSTALL_DIR}/lib
pythonpathappend \${LEGION_INSTALL_DIR}/lib/python\${PYVER}/site-packages
export CONDA_ROOT="${root_dir}/conda"
export CONDA_ENV_DIR="\$CONDA_ROOT/envs/myenv"
export CONDA_ENV_DIR="\${CONDA_ROOT}/envs/myenv"
export LCLS2_DIR="${root_dir}/lcls2"
# settings for finufft
if [[ -z \${FFTW_INC+x} ]]; then
export FINUFFT_CFLAGS="-I\$CONDA_ENV_DIR/include"
export FINUFFT_CFLAGS="-I\${CONDA_ENV_DIR}/include"
else
export FINUFFT_CFLAGS="-I\$FFTW_INC -I\$CONDA_ENV_DIR/include"
export FINUFFT_CFLAGS="-I\$FFTW_INC -I\${CONDA_ENV_DIR}/include"
fi
if [[ -z \${FFTW_DIR+x} ]]; then
export FINUFFT_LDFLAGS="-L\$CONDA_ENV_DIR/lib"
export FINUFFT_LDFLAGS="-L\${CONDA_ENV_DIR}/lib"
else
export FINUFFT_LDFLAGS="-L\$FFTW_DIR -L\$CONDA_ENV_DIR/lib"
export FINUFFT_LDFLAGS="-L\$FFTW_DIR -L\${CONDA_ENV_DIR}/lib"
fi
#cufinufft library dir
export CUFINUFFT_DIR="${root_dir}/cufinufft/lib"
ldpathappend \$CUFINUFFT_DIR
pathappend \$LCLS2_DIR/install/bin
pythonpathappend \$LCLS2_DIR/install/lib/python\$PYVER/site-packages
pathappend \${LCLS2_DIR}/install/bin
pythonpathappend \${LCLS2_DIR}/install/lib/python\${PYVER}/site-packages
if [[ -d \$CONDA_ROOT ]]; then
source "\$CONDA_ROOT/etc/profile.d/conda.sh"
conda activate "\$CONDA_ENV_DIR"
source "\${CONDA_ROOT}/etc/profile.d/conda.sh"
conda activate "\${CONDA_ENV_DIR}"
fi
EOF
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment