Image compare for verification set generation and oldest python workflow fix

.github/workflows/generate_reference_plots.yml

@@ -15,7 +15,7 @@ jobs:
           fail-fast: false
           max-parallel: 2
           matrix:
-            extras: ["all"]
+            extras: ["testpackage"]
         steps:
         - uses: actions/checkout@v4
           timeout-minutes: 5
@@ -42,9 +42,24 @@ jobs:
             DATAPATH="/wrk-kappa/group/spacephysics/analysator/CI/verification_sets/$(git rev-parse HEAD)"
             mkdir $DATAPATH
             touch $DATAPATH/.lockfile
-            sbatch -W -o testpackage_gen_verf.txt --job-name gen_verf_plots ./testpackage/run_testpackage_generate_verf_set.sh $DATAPATH
-
+            sbatch -W -o testpackage_gen_verf.txt --job-name gen_verf_plots ./testpackage/run_testpackage_generate_verf_set.sh $DATAPATH > jobid.txt
+            export JOBID=$(grep -Po '\d+' jobid.txt)
+            export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED) 
+            if [[ $SACCT_LOG ]]; then
+              echo "Some job failed on a node"
+              echo $SACCT_LOG 
+              exit 1
+            fi
             cat testpackage_gen_verf.txt
 
             python ./testpackage/testpackage_get_job_error.py testpackage_gen_verf.txt
             rm $DATAPATH/.lockfile
+        - name: Comparing plotted data
+          run: |
+            export TMPDIR=$RUNNER_TEMP
+            module load Python/3.10.4-GCCcore-11.3.0
+            . CI_env/bin/activate
+            sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh verf_set 
+            cat testpackage_compare.txt
+            python ./testpackage/testpackage_get_job_error.py testpackage_compare.txt
+

.github/workflows/test_compare_image_oldest_python.yml

@@ -36,7 +36,7 @@ jobs:
             while [[ ! $VERSIONS_MATCH ]]
             do
               echo "Trying to find module for Python/$PYTHON"
-              VERSIONS_MATCH=$(echo $AVAILABLE_VERSIONS | grep -Po 'Python/.+?-' | grep -Po 'Python'/$PYTHON)
+              VERSIONS_MATCH=$(echo "$AVAILABLE_VERSIONS" | grep -Po 'Python/.+?-GCC.+' | grep -P 'Python'/$PYTHON | head -n1)
               if [[ ! $VERSIONS_MATCH ]]; then          
                 MINOR=$(( $MINOR + 1 ))
                 PYTHON=$MAJOR.$MINOR
@@ -47,16 +47,17 @@ jobs:
                 fi
               fi
             done
-            echo "PYTHON=$PYTHON" >> $GITHUB_OUTPUT
-            module load Python/$PYTHON
+            echo "::warning::Found $VERSIONS_MATCH"
+            echo "PYTHON=$VERSIONS_MATCH" >> $GITHUB_OUTPUT
+            module load $VERSIONS_MATCH
             module list Python
         - name: Install dependencies
           timeout-minutes: 5
           run: |
             export TMPDIR=$RUNNER_TEMP
             export UV_LINK_MODE=copy
             module purge
-            module load Python/${{ steps.pyversion.outputs.PYTHON }}
+            module load ${{ steps.pyversion.outputs.PYTHON }}
             module list
             uv venv CI_env
             . CI_env/bin/activate
@@ -66,25 +67,34 @@ jobs:
           run: |
             export TMPDIR=$RUNNER_TEMP
             module purge
-            module load Python/${{ steps.pyversion.outputs.PYTHON }}
+            module load ${{ steps.pyversion.outputs.PYTHON }}
+            module list
             verf_loc="/wrk-kappa/group/spacephysics/analysator/CI/verification_sets"
             verfset=$(ls -lth $verf_loc | grep ^d | head -n1 | grep -Po '\w+$')
             if [[ -f $verf_loc/$verfset/.lockfile ]]; then 
               echo -e ".lockfile found in $verf_loc/$verfset, not running test, as the verification set generation is likely still ongoing\n Check ongoing actions and/or re-run verification set generation."
               exit 1
             fi 
-            sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh
-
+            sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh old_python ${{ steps.pyversion.outputs.PYTHON }} > jobid.txt
+            export JOBID=$(grep -Po '\d+' jobid.txt)
+            export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED) 
+            if [[ $SACCT_LOG ]]; then
+              echo "Some job failed on a node"
+              echo $SACCT_LOG 
+              exit 1
+            fi
             cat testpackage_run.txt
             . CI_env/bin/activate
             python ./testpackage/testpackage_get_job_error.py testpackage_run.txt
 
         - name: Comparing plotted data
           run: |
             export TMPDIR=$RUNNER_TEMP
-            module load Python/${{ steps.pyversion.outputs.PYTHON }}
+            module purge
+            module load ${{ steps.pyversion.outputs.PYTHON }}
+            module list
             . CI_env/bin/activate
-            sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh
+            sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh old_python ${{ steps.pyversion.outputs.PYTHON }}
             cat testpackage_compare.txt
             python ./testpackage/testpackage_get_job_error.py testpackage_compare.txt
         - name: scancel dangling job upon cancellation

.github/workflows/test_compare_images.yml

@@ -58,8 +58,14 @@ jobs:
             fi
             echo "DIFFRESULT=$DIFFRESULT" >> $GITHUB_OUTPUT
             echo "Running $DIFFRESULT"
-            sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh $DIFFRESULT
-
+            sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh $DIFFRESULT > jobid.txt
+            export JOBID=$(grep -Po '\d+' jobid.txt)
+            export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED) 
+            if [[ $SACCT_LOG ]]; then
+              echo "Some job failed on a node"
+              echo $SACCT_LOG 
+              exit 1
+            fi
             cat testpackage_run.txt
             . CI_env/bin/activate
             python ./testpackage/testpackage_get_job_error.py testpackage_run.txt

.github/workflows/test_compare_images_full.yml

@@ -49,8 +49,14 @@ jobs:
               echo -e ".lockfile found in $verf_loc/$verfset, not running test, as the verification set generation is likely still ongoing\n Check ongoing actions and/or re-run verification set generation."
               exit 1
             fi 
-            sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh
-
+            sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh > jobid.txt
+            export JOBID=$(grep -Po '\d+' jobid.txt)
+            export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED) 
+            if [[ $SACCT_LOG ]]; then
+              echo "Some job failed on a node"
+              echo $SACCT_LOG 
+              exit 1
+            fi
             cat testpackage_run.txt
             . CI_env/bin/activate
             python ./testpackage/testpackage_get_job_error.py testpackage_run.txt

testpackage/run_compare.sh

@@ -16,19 +16,24 @@ index=$(( $SLURM_ARRAY_TASK_ID - $SLURM_ARRAY_TASK_MIN ))
 hostname
 
 module purge
-module load Python/3.10.4-GCCcore-11.3.0
+if [[ $1 == 'old_python' ]]; then
+  module load $2
+else
+  module load Python/3.10.4-GCCcore-11.3.0
+fi
 source CI_env/bin/activate
 module load libglvnd/1.7.0-GCCcore-13.3.0
 module list
 
+echo "SLURM_JOB_ID=$SLURM_ARRAY_JOB_ID" >> $GITHUB_OUTPUT
 verf_loc="/wrk-kappa/group/spacephysics/analysator/CI/verification_sets"
 
 #if pass we do not check for anything
 if echo $@ | grep -q -P "\spass$|\spass\s|pass"; then
    exit 0
 fi
 
-check=true
+check=false
 
 #gets latest verfication set (based on modification date -> grep directories only -> take firstline -> get last word)
 verfset=$(ls -lth $verf_loc | grep ^d | head -n1 | grep -Po '\w+$')
@@ -37,23 +42,42 @@ if [[ -f $verf_loc/$verfset/.lockfile ]]; then
   echo ".lockfile found in $verf_loc/$verfset, not comparing, something probably went wrong removing the lockfile"
   exit 1
 fi
+
+#Do verf_set compare 
+if [[ $@ == 'verf_set' ]]; then
+  verfset2=$(ls -lht $verf_loc | grep ^d | sed -n 2p | grep -Po '\w+$')
+  echo "Comparing two latest verification sets $verfset and $verfset2"
+  folder_1="$verf_loc/$verfset/" 
+  folder_2="$verf_loc/$verfset2/"
+  python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index 0 && echo "No differences found" 
+  echo "EXIT_CODE_FROM_JOB $?"
+  exit 0
+#If old_python is used, run all
+elif [[ $1 == 'old_python' ]]; then
+  check=false
+#Do selective compare if other arguments
+elif [ $@ ]; then
+  check=true
+fi
+
 echo "Comparing against $verfset"
-#Note that this is skipped if no arguments are passed
-for i in $@
-do
-    check=false
-    echo "Comparing for $i"
-    folder_1="$verf_loc/$verfset/$i/" 
-    folder_2="${PWD}/produced_plots/$i/"
-    python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index && echo "No differences found in produced images"
-    echo "EXIT_CODE_FROM_JOB $?"
-done
+
 
 if $check;
 then
-    echo "Comparing all"
-    folder_1="$verf_loc/$verfset/" 
-    folder_2="${PWD}/produced_plots/"
-    python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index && echo "No differences found in produced images"
-    echo "EXIT_CODE_FROM_JOB $?"
+  for i in $@
+  do
+      echo "Comparing for $i"
+      folder_1="$verf_loc/$verfset/$i/" 
+      folder_2="${PWD}/produced_plots/$i/"
+      python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index && echo "No differences found in produced images"
+      echo "EXIT_CODE_FROM_JOB $?"
+  done
+  exit 0
+else
+  echo "Comparing all"
+  folder_1="$verf_loc/$verfset/" 
+  folder_2="${PWD}/produced_plots/"
+  python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index && echo "No differences found in produced images"
+  echo "EXIT_CODE_FROM_JOB $?"
 fi

testpackage/run_testpackage_generate_verf_set.sh

@@ -13,6 +13,7 @@ index=$(( $SLURM_ARRAY_TASK_ID - $SLURM_ARRAY_TASK_MIN ))
 
 hostname
 
+echo "SLURM_JOB_ID=$SLURM_ARRAY_JOB_ID" >> $GITHUB_OUTPUT
 source CI_env/bin/activate
 export PATH=/wrk-vakka/group/spacephysics/proj/appl/tex-basic/texlive/2023/bin/x86_64-linux:$PATH
 

testpackage/run_testpackage_workflow.sh

@@ -19,9 +19,14 @@ hostname
 
 module purge            
 export PATH=/wrk-vakka/group/spacephysics/proj/appl/tex-basic/texlive/2023/bin/x86_64-linux:$PATH
-
+older_python=false
 echo "SLURM_JOB_ID=$SLURM_ARRAY_JOB_ID" >> $GITHUB_OUTPUT
-module load Python/3.10.4-GCCcore-11.3.0
+if [[ $1 == "old_python" ]]; then
+  module load $2
+  older_python=true
+else
+  module load Python/3.10.4-GCCcore-11.3.0
+fi
 source CI_env/bin/activate
 
 module list
@@ -32,6 +37,11 @@ export PTNONINTERACTIVE=1
 export PTOUTPUTDIR=$PWD/produced_plots/
 
 
-
+if $older_python; then
+  echo "::warning:: Running with older python version $2"
+  python ./testpackage/testpackage_commons.py $jobcount $index 
+  echo "EXIT_CODE_FROM_JOB $?"
+  exit 0
+fi
 python ./testpackage/testpackage_commons.py $jobcount $index $@
 echo "EXIT_CODE_FROM_JOB $?"

testpackage/testpackage_compare.py

@@ -11,7 +11,7 @@ def compare_images(a,b):
     return im1.shape == im2.shape and not(np.bitwise_xor(im1,im2).any())
 
 
-def compare_images_in_folders(a,b,jobcount,jobcurr):
+def compare_images_in_folders(a,b,jobcount,jobcurr,error_mode):
 
     #do the comparisons
 
@@ -68,7 +68,6 @@ def compare_images_in_folders(a,b,jobcount,jobcurr):
         except Exception as e:
             raise SystemError(f"Error calling compare_images({file},{file.replace(a,b)}):"+str(e))
 
-
     #Print unique and missing files
     if jobcurr==0:
         for file in unique_files:
@@ -81,13 +80,17 @@ def compare_images_in_folders(a,b,jobcount,jobcurr):
             print("::warning::Found new file(s) produced by the code!")
 
         if len(missing_files)!=0:
-            raise SystemError("Found file(s) **not** produced by the code!")
-
+            if error_mode:
+                raise SystemError("Found file(s) **not** produced by the code!")
+            elif not error_mode:
+                print("::warning::Found file(s) **not** produced by the code!")
 
     if different:
-        print(f"::error title=Plot(s) differ::Produced plots not in agreement with the verfication set {a}")
-        raise SystemError("Images Differ")
-
+        if error_mode:
+            print(f"::error title=Plot(s) differ::Produced plots not in agreement with the verfication set {a}")
+            raise SystemError("Images Differ")
+        elif not error_mode:
+            print(f"::warning title=Plot(s) differ::Produced new verification set {a} not in agreement with the old verfication set {b}")
 
 
 if __name__=='__main__':
@@ -102,13 +105,13 @@ def compare_images_in_folders(a,b,jobcount,jobcurr):
 
     parser.add_argument("jobcount",help="Number of parallel jobs to use",default=1,nargs='?',type=int)
     parser.add_argument("jobindex",help="Index of the job to run",default=0,nargs='?',type=int)
-
+    parser.add_argument("errormode",help="handle differing files as errors.",default=1,nargs='?',type=int) 
 
     args= parser.parse_args()
     a,b = args.folder_a,args.folder_b
-
+    error_mode=args.errormode
     jobcount=args.jobcount
     jobindex=args.jobindex
-
-    compare_images_in_folders(a,b,jobcount,jobindex)
+    error_mode=bool(error_mode)
+    compare_images_in_folders(a,b,jobcount,jobindex,error_mode)