spinifel IndexError on one node
1-node spinifel run fails with "IndexError: index 0 is out of bounds". This happened before and after I put in the changes Iris said for autocorrelation.py. It happens in multiple configurations with 1 node. These are my code changes. Also attaching a couple log files that show it.
Not sure who to assign it to, so please change it if needed.
[cahrens@login4.summit spinifel-devel-070821]$ git diff
diff --git a/settings/summit_quickstart.toml b/settings/summit_quickstart.toml
index c16691a..4426aee 100644
--- a/settings/summit_quickstart.toml
+++ b/settings/summit_quickstart.toml
@@ -8,7 +8,6 @@ use_callmonitor = false
[data]
in_dir = "/gpfs/alpine/proj-shared/chm137/data/spi"
-out_dir = "/gpfs/alpine/proj-shared/chm137/${USER}/spinifel_output"
name = "2CEX-10k-2.h5"
[psana]
@@ -18,12 +17,10 @@ ps_eb_nodes = 1
ps_srv_nodes = 0
[runtime]
-n_images_per_rank = 10
-small_problem = true
using_cuda = true
use_cufinufft = true
use_single_prec = false
-chk_convergence = true
+chk_convergence = false
[gpu]
devices_per_node = 1
diff --git a/setup/finufft b/setup/finufft
--- a/setup/finufft
+++ b/setup/finufft
@@ -1 +1 @@
-Subproject commit fb3acac2131e833b20f96ee662fe43b2030c89d4
+Subproject commit fb3acac2131e833b20f96ee662fe43b2030c89d4-dirty
diff --git a/setup/legion b/setup/legion
--- a/setup/legion
+++ b/setup/legion
@@ -1 +1 @@
-Subproject commit c71186a97a176f9bcf0c5e4cbf9583b543751714
+Subproject commit c71186a97a176f9bcf0c5e4cbf9583b543751714-dirty
diff --git a/spinifel/mpi/autocorrelation.py b/spinifel/mpi/autocorrelation.py
index 9c10fc9..a27a757 100644
--- a/spinifel/mpi/autocorrelation.py
+++ b/spinifel/mpi/autocorrelation.py
@@ -157,7 +157,8 @@ def solve_ac(generation,
weights = np.ones(N)
# remove M**3 in the numerator
- rlambda = 1./Ntot * 10**(comm.rank - comm.size/2)
+ rlambda = np.logspace(-8, 8, comm.size)[comm.rank]
+ #rlambda = 1./Ntot * 10**(comm.rank - comm.size/2)
flambda = 0 # 1e5 * pow(10, comm.rank - comm.size//2)
maxiter = parms.solve_ac_maxiter
diff --git a/spinifel/mpi/main.py b/spinifel/mpi/main.py
index 50d6c34..5b45fe9 100644
--- a/spinifel/mpi/main.py
+++ b/spinifel/mpi/main.py
@@ -52,7 +52,15 @@ def main():
logger.log(f"writerrank: {writer_rank}")
logger.log(f"batch_size: {batch_size}")
logger.log(f"max_events: {max_events}")
-
+ logger.log(f"parms.N_binning: {parms.N_binning}")
+ logger.log(f"parms.nER: {parms.nER}")
+ logger.log(f"parms.nHIO: {parms.nHIO}")
+ logger.log(f"parms.N_phase_loops: {parms.N_phase_loops}")
+ logger.log(f"parms.N_clipping: {parms.N_clipping}")
+ logger.log(f"parms.N_binning: {parms.N_binning}")
+ logger.log(f"parms.N_orientations: {parms.N_orientations}")
+ logger.log(f"parms.N_batch_size: {parms.N_batch_size}")
+ logger.log(f"parms.Mquat: {parms.Mquat}")
# Load unique set of intensity slices for each rank
(pixel_position_reciprocal,
diff --git a/spinifel/parms.py b/spinifel/parms.py
index ce2cd05..75369ac 100644
--- a/spinifel/parms.py
+++ b/spinifel/parms.py
@@ -60,8 +60,8 @@ else:
nHIO = 25
N_phase_loops = 10
N_clipping = 0
- N_binning = 0
- N_orientations = 3000 # model_slices
+ N_binning = 3
+ N_orientations = 2000 # model_slices
N_batch_size = 100
Mquat = int(oversampling * 20) # 1/4 of uniform grid size