From 612fa5f471cb0d061f8bb24a964dd69eb448fd42 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Thu, 8 Jan 2026 10:23:56 +0200
Subject: [PATCH 01/28] Preliminary compare in gen verf

---
 .github/workflows/generate_reference_plots.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.github/workflows/generate_reference_plots.yml b/.github/workflows/generate_reference_plots.yml
index f5e54f53..ea375dd2 100644
--- a/.github/workflows/generate_reference_plots.yml
+++ b/.github/workflows/generate_reference_plots.yml
@@ -48,3 +48,12 @@ jobs:
 
             python ./testpackage/testpackage_get_job_error.py testpackage_gen_verf.txt
             rm $DATAPATH/.lockfile
+        - name: Comparing plotted data
+          run: |
+            export TMPDIR=$RUNNER_TEMP
+            module load Python/3.10.4-GCCcore-11.3.0
+            . CI_env/bin/activate
+            sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh 
+            cat testpackage_compare.txt
+            python ./testpackage/testpackage_get_job_error.py testpackage_compare.txt
+     

From 8effbdcb4f2c9a2c8d9b9da13a8fd64a98d1eb99 Mon Sep 17 00:00:00 2001
From: lassejsc <lasse.b.sihvonen@helsinki.fi>
Date: Thu, 8 Jan 2026 12:19:39 +0200
Subject: [PATCH 02/28] Adding checking of two of the latest verification sets
 to generate verification set workflow

---
 .../workflows/generate_reference_plots.yml    |  2 +-
 testpackage/run_compare.sh                    | 48 ++++++++++++-------
 2 files changed, 33 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/generate_reference_plots.yml b/.github/workflows/generate_reference_plots.yml
index ea375dd2..ce3d46f5 100644
--- a/.github/workflows/generate_reference_plots.yml
+++ b/.github/workflows/generate_reference_plots.yml
@@ -53,7 +53,7 @@ jobs:
             export TMPDIR=$RUNNER_TEMP
             module load Python/3.10.4-GCCcore-11.3.0
             . CI_env/bin/activate
-            sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh 
+            sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh verf_set 
             cat testpackage_compare.txt
             python ./testpackage/testpackage_get_job_error.py testpackage_compare.txt
      
diff --git a/testpackage/run_compare.sh b/testpackage/run_compare.sh
index dd3a15f9..3a99ad4c 100755
--- a/testpackage/run_compare.sh
+++ b/testpackage/run_compare.sh
@@ -28,7 +28,7 @@ if echo $@ | grep -q -P "\spass$|\spass\s|pass"; then
    exit 0
 fi
 
-check=true
+check=false
 
 #gets latest verfication set (based on modification date -> grep directories only -> take firstline -> get last word)
 verfset=$(ls -lth $verf_loc | grep ^d | head -n1 | grep -Po '\w+$')
@@ -38,22 +38,38 @@ if [[ -f $verf_loc/$verfset/.lockfile ]]; then
   exit 1
 fi
 echo "Comparing against $verfset"
-#Note that this is skipped if no arguments are passed
-for i in $@
-do
-    check=false
-    echo "Comparing for $i"
-    folder_1="$verf_loc/$verfset/$i/" 
-    folder_2="${PWD}/produced_plots/$i/"
-    python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index && echo "No differences found in produced images"
-    echo "EXIT_CODE_FROM_JOB $?"
-done
+
+#Do verf_set compare 
+if [ $@ == 'verf_set' ]; then
+  verfset2=$(ls -lht | grep ^d | sed -n 2p | grep -Po '\w+$')
+  echo "Comparing two latest verification sets"
+  folder_1="$verf_loc/$verfset/" 
+  folder_2="$verf_loc/$verfset2/"
+  python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index && echo "No differences found in produced images"
+  echo "EXIT_CODE_FROM_JOB $?"
+  exit 0
+#Do selective compare if other arguments
+elif [ $@ ]; then
+  check=true
+fi
+
+
 
 if $check;
 then
-    echo "Comparing all"
-    folder_1="$verf_loc/$verfset/" 
-    folder_2="${PWD}/produced_plots/"
-    python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index && echo "No differences found in produced images"
-    echo "EXIT_CODE_FROM_JOB $?"
+  for i in $@
+  do
+      echo "Comparing for $i"
+      folder_1="$verf_loc/$verfset/$i/" 
+      folder_2="${PWD}/produced_plots/$i/"
+      python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index && echo "No differences found in produced images"
+      echo "EXIT_CODE_FROM_JOB $?"
+  done
+  exit 0
+else
+  echo "Comparing all"
+  folder_1="$verf_loc/$verfset/" 
+  folder_2="${PWD}/produced_plots/"
+  python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index && echo "No differences found in produced images"
+  echo "EXIT_CODE_FROM_JOB $?"
 fi

From 0742c4b8a3fe3effcac67aaeee8d3e3c25a6aafc Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Thu, 8 Jan 2026 12:59:28 +0200
Subject: [PATCH 03/28] Set the analysator flag to testpackage on
 generate_reference_plots

---
 .github/workflows/generate_reference_plots.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/generate_reference_plots.yml b/.github/workflows/generate_reference_plots.yml
index ce3d46f5..633f91c8 100644
--- a/.github/workflows/generate_reference_plots.yml
+++ b/.github/workflows/generate_reference_plots.yml
@@ -15,7 +15,7 @@ jobs:
           fail-fast: false
           max-parallel: 2
           matrix:
-            extras: ["all"]
+            extras: ["testpackage"]
         steps:
         - uses: actions/checkout@v4
           timeout-minutes: 5

From ac992a855132ddf6996510c51dda38fa52b5e9a2 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Thu, 8 Jan 2026 14:23:40 +0200
Subject: [PATCH 04/28] added correct path to ls

---
 testpackage/run_compare.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testpackage/run_compare.sh b/testpackage/run_compare.sh
index 3a99ad4c..b44c692c 100755
--- a/testpackage/run_compare.sh
+++ b/testpackage/run_compare.sh
@@ -41,7 +41,7 @@ echo "Comparing against $verfset"
 
 #Do verf_set compare 
 if [ $@ == 'verf_set' ]; then
-  verfset2=$(ls -lht | grep ^d | sed -n 2p | grep -Po '\w+$')
+  verfset2=$(ls -lht $verf_loc | grep ^d | sed -n 2p | grep -Po '\w+$')
   echo "Comparing two latest verification sets"
   folder_1="$verf_loc/$verfset/" 
   folder_2="$verf_loc/$verfset2/"

From aec121ecbed35bf188f4d816395b4bcc2229bbd3 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Thu, 8 Jan 2026 14:27:25 +0200
Subject: [PATCH 05/28] Added better echo to compare

---
 testpackage/run_compare.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/testpackage/run_compare.sh b/testpackage/run_compare.sh
index b44c692c..9bd623b8 100755
--- a/testpackage/run_compare.sh
+++ b/testpackage/run_compare.sh
@@ -37,12 +37,11 @@ if [[ -f $verf_loc/$verfset/.lockfile ]]; then
   echo ".lockfile found in $verf_loc/$verfset, not comparing, something probably went wrong removing the lockfile"
   exit 1
 fi
-echo "Comparing against $verfset"
 
 #Do verf_set compare 
 if [ $@ == 'verf_set' ]; then
   verfset2=$(ls -lht $verf_loc | grep ^d | sed -n 2p | grep -Po '\w+$')
-  echo "Comparing two latest verification sets"
+  echo "Comparing two latest verification sets $verfset and $verfset2"
   folder_1="$verf_loc/$verfset/" 
   folder_2="$verf_loc/$verfset2/"
   python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index && echo "No differences found in produced images"
@@ -53,6 +52,7 @@ elif [ $@ ]; then
   check=true
 fi
 
+echo "Comparing against $verfset"
 
 
 if $check;

From 96392a30073473a5af4fbada62a427f4e4568984 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Thu, 8 Jan 2026 15:31:51 +0200
Subject: [PATCH 06/28] Added way to make errors to warnings in compare

---
 testpackage/testpackage_compare.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/testpackage/testpackage_compare.py b/testpackage/testpackage_compare.py
index ed66863c..ce40e398 100644
--- a/testpackage/testpackage_compare.py
+++ b/testpackage/testpackage_compare.py
@@ -11,7 +11,7 @@ def compare_images(a,b):
     return im1.shape == im2.shape and not(np.bitwise_xor(im1,im2).any())
 
 
-def compare_images_in_folders(a,b,jobcount,jobcurr):
+def compare_images_in_folders(a,b,jobcount,jobcurr,error_mode):
 
     #do the comparisons
     
@@ -81,13 +81,17 @@ def compare_images_in_folders(a,b,jobcount,jobcurr):
             print("::warning::Found new file(s) produced by the code!")
 
         if len(missing_files)!=0:
-            raise SystemError("Found file(s) **not** produced by the code!")
-
+            if error_mode:
+                raise SystemError("Found file(s) **not** produced by the code!")
+            elif not error_mode:
+                print("::warning::Found file(s) **not** produced by the code!")
 
     if different:
-        print(f"::error title=Plot(s) differ::Produced plots not in agreement with the verfication set {a}")
-        raise SystemError("Images Differ")
-
+        if error_mode:
+            print(f"::error title=Plot(s) differ::Produced plots not in agreement with the verfication set {a}")
+            raise SystemError("Images Differ")
+        elif not error_mode:
+            print(f"::warning title=Plot(s) differ::Produced new verification set {a} not in agreement with the old verfication set {b}")
 
 
 if __name__=='__main__':
@@ -102,13 +106,13 @@ def compare_images_in_folders(a,b,jobcount,jobcurr):
    
     parser.add_argument("jobcount",help="Number of parallel jobs to use",default=1,nargs='?',type=int)
     parser.add_argument("jobindex",help="Index of the job to run",default=0,nargs='?',type=int)
-    
+    parser.add_argument("error_mode",help="handle differing files as errors.",default=True,nargs='?',type=bool) 
    
     args= parser.parse_args()
     a,b = args.folder_a,args.folder_b
-
+    error_mode=args.error_mode
     jobcount=args.jobcount
     jobindex=args.jobindex
 
-    compare_images_in_folders(a,b,jobcount,jobindex)
+    compare_images_in_folders(a,b,jobcount,jobindex,error_mode)
     

From 9468b83656fc8674aada6ed5e6e3cbe26aa047f5 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Thu, 8 Jan 2026 15:33:35 +0200
Subject: [PATCH 07/28] added the correct error mode to run_compare verf_set
 compare

---
 testpackage/run_compare.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testpackage/run_compare.sh b/testpackage/run_compare.sh
index 9bd623b8..3195154f 100755
--- a/testpackage/run_compare.sh
+++ b/testpackage/run_compare.sh
@@ -62,7 +62,7 @@ then
       echo "Comparing for $i"
       folder_1="$verf_loc/$verfset/$i/" 
       folder_2="${PWD}/produced_plots/$i/"
-      python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index && echo "No differences found in produced images"
+      python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index False && echo "No differences found in produced images"
       echo "EXIT_CODE_FROM_JOB $?"
   done
   exit 0

From 3002ccc9e2017f41224337ce50a8f3a61c94c668 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Thu, 8 Jan 2026 16:33:15 +0200
Subject: [PATCH 08/28] Moved the error mode argument to correct spot for
 verification set generation

---
 testpackage/run_compare.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/testpackage/run_compare.sh b/testpackage/run_compare.sh
index 3195154f..6572ad12 100755
--- a/testpackage/run_compare.sh
+++ b/testpackage/run_compare.sh
@@ -44,7 +44,7 @@ if [ $@ == 'verf_set' ]; then
   echo "Comparing two latest verification sets $verfset and $verfset2"
   folder_1="$verf_loc/$verfset/" 
   folder_2="$verf_loc/$verfset2/"
-  python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index && echo "No differences found in produced images"
+  python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index False && echo "No differences found in produced images"
   echo "EXIT_CODE_FROM_JOB $?"
   exit 0
 #Do selective compare if other arguments
@@ -62,7 +62,7 @@ then
       echo "Comparing for $i"
       folder_1="$verf_loc/$verfset/$i/" 
       folder_2="${PWD}/produced_plots/$i/"
-      python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index False && echo "No differences found in produced images"
+      python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index && echo "No differences found in produced images"
       echo "EXIT_CODE_FROM_JOB $?"
   done
   exit 0

From f5d305211784f4675996ad364850cd034576a9c6 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Thu, 8 Jan 2026 17:18:58 +0200
Subject: [PATCH 09/28] Argument parses did not want to work with booleans, ok
 very cool

---
 testpackage/run_compare.sh         | 2 +-
 testpackage/testpackage_compare.py | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/testpackage/run_compare.sh b/testpackage/run_compare.sh
index 6572ad12..015d0dd4 100755
--- a/testpackage/run_compare.sh
+++ b/testpackage/run_compare.sh
@@ -44,7 +44,7 @@ if [ $@ == 'verf_set' ]; then
   echo "Comparing two latest verification sets $verfset and $verfset2"
   folder_1="$verf_loc/$verfset/" 
   folder_2="$verf_loc/$verfset2/"
-  python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index False && echo "No differences found in produced images"
+  python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index 0 && echo "No differences found" 
   echo "EXIT_CODE_FROM_JOB $?"
   exit 0
 #Do selective compare if other arguments
diff --git a/testpackage/testpackage_compare.py b/testpackage/testpackage_compare.py
index ce40e398..cda3dbba 100644
--- a/testpackage/testpackage_compare.py
+++ b/testpackage/testpackage_compare.py
@@ -68,7 +68,6 @@ def compare_images_in_folders(a,b,jobcount,jobcurr,error_mode):
         except Exception as e:
             raise SystemError(f"Error calling compare_images({file},{file.replace(a,b)}):"+str(e))
 
-
     #Print unique and missing files
     if jobcurr==0:
         for file in unique_files:
@@ -106,13 +105,13 @@ def compare_images_in_folders(a,b,jobcount,jobcurr,error_mode):
    
     parser.add_argument("jobcount",help="Number of parallel jobs to use",default=1,nargs='?',type=int)
     parser.add_argument("jobindex",help="Index of the job to run",default=0,nargs='?',type=int)
-    parser.add_argument("error_mode",help="handle differing files as errors.",default=True,nargs='?',type=bool) 
+    parser.add_argument("errormode",help="handle differing files as errors.",default=1,nargs='?',type=int) 
    
     args= parser.parse_args()
     a,b = args.folder_a,args.folder_b
-    error_mode=args.error_mode
+    error_mode=args.errormode
     jobcount=args.jobcount
     jobindex=args.jobindex
-
+    error_mode=bool(error_mode)
     compare_images_in_folders(a,b,jobcount,jobindex,error_mode)
     

From 9a656e0467239a2cf0119c1f51993c2a6befdcb0 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Fri, 9 Jan 2026 09:45:13 +0200
Subject: [PATCH 10/28] Added a check for silently failing nodes

---
 testpackage/run_compare.sh                     |  2 +-
 .../run_testpackage_generate_verf_set.sh       |  2 +-
 testpackage/run_testpackage_workflow.sh        |  3 +--
 testpackage/testpackage_get_job_error.py       | 18 ++++++++++++++----
 4 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/testpackage/run_compare.sh b/testpackage/run_compare.sh
index 015d0dd4..193141dc 100755
--- a/testpackage/run_compare.sh
+++ b/testpackage/run_compare.sh
@@ -13,7 +13,7 @@
 jobcount=$(( $SLURM_ARRAY_TASK_MAX - $SLURM_ARRAY_TASK_MIN + 1 )) 
 index=$(( $SLURM_ARRAY_TASK_ID - $SLURM_ARRAY_TASK_MIN ))
 
-hostname
+echo "HOSTNAME $(hostname) 10"
 
 module purge
 module load Python/3.10.4-GCCcore-11.3.0
diff --git a/testpackage/run_testpackage_generate_verf_set.sh b/testpackage/run_testpackage_generate_verf_set.sh
index 39f274b9..7a39c9f1 100644
--- a/testpackage/run_testpackage_generate_verf_set.sh
+++ b/testpackage/run_testpackage_generate_verf_set.sh
@@ -11,7 +11,7 @@
 jobcount=$(( $SLURM_ARRAY_TASK_MAX - $SLURM_ARRAY_TASK_MIN + 1 )) 
 index=$(( $SLURM_ARRAY_TASK_ID - $SLURM_ARRAY_TASK_MIN ))
 
-hostname
+echo "HOSTNAME $(hostname) 11"
 
 source CI_env/bin/activate
 export PATH=/wrk-vakka/group/spacephysics/proj/appl/tex-basic/texlive/2023/bin/x86_64-linux:$PATH
diff --git a/testpackage/run_testpackage_workflow.sh b/testpackage/run_testpackage_workflow.sh
index 9eaa6e96..016955e9 100755
--- a/testpackage/run_testpackage_workflow.sh
+++ b/testpackage/run_testpackage_workflow.sh
@@ -14,8 +14,7 @@ jobcount=$(( $SLURM_ARRAY_TASK_MAX - $SLURM_ARRAY_TASK_MIN + 1 ))
 index=$(( $SLURM_ARRAY_TASK_ID - $SLURM_ARRAY_TASK_MIN ))
 
 
-hostname
-
+echo "HOSTNAME $(hostname) 20"
 
 module purge            
 export PATH=/wrk-vakka/group/spacephysics/proj/appl/tex-basic/texlive/2023/bin/x86_64-linux:$PATH
diff --git a/testpackage/testpackage_get_job_error.py b/testpackage/testpackage_get_job_error.py
index 9a4aa5fb..ad8fdfc6 100644
--- a/testpackage/testpackage_get_job_error.py
+++ b/testpackage/testpackage_get_job_error.py
@@ -13,11 +13,21 @@
 
 file= args.log_file
 
+hostname_count=0
+hostname_target=-1
+
+
 with open(file,'r') as f:
     lines = f.readlines()
     for line in lines:
-        error_code = line.split(" ")
-        if error_code[0]=="EXIT_CODE_FROM_JOB" and error_code[1].rstrip("\n")!="0":
-            raise SystemExit(int(line.split(" ")[1].rstrip("\n")))
-
+        line_list = line.split(" ")
+        if line_list[0]=="EXIT_CODE_FROM_JOB" and line_list[1].rstrip("\n")!="0":
+            raise SystemExit(int(line_list[1].rstrip("\n")))
+        elif line_list[0]=="HOSTNAME":
+            hostname_target=int(line_list[2].rstrip("\n"))
+            hostname_count+=1
+    f.close()
+
+if hostname_target!=hostname_count:
+    raise SystemExit(f"::error::{hostname_target-hostname_count} node(s) likely failed silently")
 

From 19d2bcce460e550f88d2d41fb8e5f3912ba64259 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Fri, 9 Jan 2026 10:12:35 +0200
Subject: [PATCH 11/28] Added missing []

---
 testpackage/run_compare.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testpackage/run_compare.sh b/testpackage/run_compare.sh
index 193141dc..fcbb313d 100755
--- a/testpackage/run_compare.sh
+++ b/testpackage/run_compare.sh
@@ -39,7 +39,7 @@ if [[ -f $verf_loc/$verfset/.lockfile ]]; then
 fi
 
 #Do verf_set compare 
-if [ $@ == 'verf_set' ]; then
+if [[ $@ == 'verf_set' ]]; then
   verfset2=$(ls -lht $verf_loc | grep ^d | sed -n 2p | grep -Po '\w+$')
   echo "Comparing two latest verification sets $verfset and $verfset2"
   folder_1="$verf_loc/$verfset/" 

From b793cf9c0326f7974946db6f9c619b500c94090c Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Fri, 9 Jan 2026 11:00:57 +0200
Subject: [PATCH 12/28] Revert "Added a check for silently failing nodes"

This reverts commit 9a656e0467239a2cf0119c1f51993c2a6befdcb0.
---
 testpackage/run_compare.sh                     |  2 +-
 .../run_testpackage_generate_verf_set.sh       |  2 +-
 testpackage/run_testpackage_workflow.sh        |  3 ++-
 testpackage/testpackage_get_job_error.py       | 18 ++++--------------
 4 files changed, 8 insertions(+), 17 deletions(-)

diff --git a/testpackage/run_compare.sh b/testpackage/run_compare.sh
index fcbb313d..6d52ac9a 100755
--- a/testpackage/run_compare.sh
+++ b/testpackage/run_compare.sh
@@ -13,7 +13,7 @@
 jobcount=$(( $SLURM_ARRAY_TASK_MAX - $SLURM_ARRAY_TASK_MIN + 1 )) 
 index=$(( $SLURM_ARRAY_TASK_ID - $SLURM_ARRAY_TASK_MIN ))
 
-echo "HOSTNAME $(hostname) 10"
+hostname
 
 module purge
 module load Python/3.10.4-GCCcore-11.3.0
diff --git a/testpackage/run_testpackage_generate_verf_set.sh b/testpackage/run_testpackage_generate_verf_set.sh
index 7a39c9f1..39f274b9 100644
--- a/testpackage/run_testpackage_generate_verf_set.sh
+++ b/testpackage/run_testpackage_generate_verf_set.sh
@@ -11,7 +11,7 @@
 jobcount=$(( $SLURM_ARRAY_TASK_MAX - $SLURM_ARRAY_TASK_MIN + 1 )) 
 index=$(( $SLURM_ARRAY_TASK_ID - $SLURM_ARRAY_TASK_MIN ))
 
-echo "HOSTNAME $(hostname) 11"
+hostname
 
 source CI_env/bin/activate
 export PATH=/wrk-vakka/group/spacephysics/proj/appl/tex-basic/texlive/2023/bin/x86_64-linux:$PATH
diff --git a/testpackage/run_testpackage_workflow.sh b/testpackage/run_testpackage_workflow.sh
index 016955e9..9eaa6e96 100755
--- a/testpackage/run_testpackage_workflow.sh
+++ b/testpackage/run_testpackage_workflow.sh
@@ -14,7 +14,8 @@ jobcount=$(( $SLURM_ARRAY_TASK_MAX - $SLURM_ARRAY_TASK_MIN + 1 ))
 index=$(( $SLURM_ARRAY_TASK_ID - $SLURM_ARRAY_TASK_MIN ))
 
 
-echo "HOSTNAME $(hostname) 20"
+hostname
+
 
 module purge            
 export PATH=/wrk-vakka/group/spacephysics/proj/appl/tex-basic/texlive/2023/bin/x86_64-linux:$PATH
diff --git a/testpackage/testpackage_get_job_error.py b/testpackage/testpackage_get_job_error.py
index ad8fdfc6..9a4aa5fb 100644
--- a/testpackage/testpackage_get_job_error.py
+++ b/testpackage/testpackage_get_job_error.py
@@ -13,21 +13,11 @@
 
 file= args.log_file
 
-hostname_count=0
-hostname_target=-1
-
-
 with open(file,'r') as f:
     lines = f.readlines()
     for line in lines:
-        line_list = line.split(" ")
-        if line_list[0]=="EXIT_CODE_FROM_JOB" and line_list[1].rstrip("\n")!="0":
-            raise SystemExit(int(line_list[1].rstrip("\n")))
-        elif line_list[0]=="HOSTNAME":
-            hostname_target=int(line_list[2].rstrip("\n"))
-            hostname_count+=1
-    f.close()
-
-if hostname_target!=hostname_count:
-    raise SystemExit(f"::error::{hostname_target-hostname_count} node(s) likely failed silently")
+        error_code = line.split(" ")
+        if error_code[0]=="EXIT_CODE_FROM_JOB" and error_code[1].rstrip("\n")!="0":
+            raise SystemExit(int(line.split(" ")[1].rstrip("\n")))
+
 

From 251b5f0e1c8ed34f53ddd6247350d3b509d572c5 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Fri, 9 Jan 2026 11:16:14 +0200
Subject: [PATCH 13/28] Added SLURM_JOB_ID output to other run scripts and
 testing whether it works with sacct in workflow for finding failing nodes

---
 .github/workflows/test_compare_images.yml        | 3 ++-
 testpackage/run_compare.sh                       | 1 +
 testpackage/run_testpackage_generate_verf_set.sh | 1 +
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_compare_images.yml b/.github/workflows/test_compare_images.yml
index 3ba2a682..39af7477 100644
--- a/.github/workflows/test_compare_images.yml
+++ b/.github/workflows/test_compare_images.yml
@@ -59,7 +59,8 @@ jobs:
             echo "DIFFRESULT=$DIFFRESULT" >> $GITHUB_OUTPUT
             echo "Running $DIFFRESULT"
             sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh $DIFFRESULT
-
+            echo "JOBID=$SLURM_JOB_ID"
+            sacct -j $SLURM_JOB_ID -o job,state,node | grep FAILED
             cat testpackage_run.txt
             . CI_env/bin/activate
             python ./testpackage/testpackage_get_job_error.py testpackage_run.txt
diff --git a/testpackage/run_compare.sh b/testpackage/run_compare.sh
index 6d52ac9a..24ce3d71 100755
--- a/testpackage/run_compare.sh
+++ b/testpackage/run_compare.sh
@@ -21,6 +21,7 @@ source CI_env/bin/activate
 module load libglvnd/1.7.0-GCCcore-13.3.0
 module list
 
+echo "SLURM_JOB_ID=$SLURM_ARRAY_JOB_ID" >> $GITHUB_OUTPUT
 verf_loc="/wrk-kappa/group/spacephysics/analysator/CI/verification_sets"
 
 #if pass we do not check for anything
diff --git a/testpackage/run_testpackage_generate_verf_set.sh b/testpackage/run_testpackage_generate_verf_set.sh
index 39f274b9..f08b77ed 100644
--- a/testpackage/run_testpackage_generate_verf_set.sh
+++ b/testpackage/run_testpackage_generate_verf_set.sh
@@ -13,6 +13,7 @@ index=$(( $SLURM_ARRAY_TASK_ID - $SLURM_ARRAY_TASK_MIN ))
 
 hostname
 
+echo "SLURM_JOB_ID=$SLURM_ARRAY_JOB_ID" >> $GITHUB_OUTPUT
 source CI_env/bin/activate
 export PATH=/wrk-vakka/group/spacephysics/proj/appl/tex-basic/texlive/2023/bin/x86_64-linux:$PATH
 

From 6280a5672e7d0d139264f871f33e0ad5a04d50bc Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Fri, 9 Jan 2026 11:44:54 +0200
Subject: [PATCH 14/28] Fixed the SLURM_JOB_ID env var , testing

---
 .github/workflows/test_compare_images.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_compare_images.yml b/.github/workflows/test_compare_images.yml
index 39af7477..2d139f0d 100644
--- a/.github/workflows/test_compare_images.yml
+++ b/.github/workflows/test_compare_images.yml
@@ -59,8 +59,9 @@ jobs:
             echo "DIFFRESULT=$DIFFRESULT" >> $GITHUB_OUTPUT
             echo "Running $DIFFRESULT"
             sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh $DIFFRESULT
-            echo "JOBID=$SLURM_JOB_ID"
-            sacct -j $SLURM_JOB_ID -o job,state,node | grep FAILED
+
+            echo ${{ steps.run_cl.outputs.SLURM_JOB_ID }}
+            sacct -j ${{ steps.run_cl.outputs.SLURM_JOB_ID }} -o job,state,node | grep FAILED
             cat testpackage_run.txt
             . CI_env/bin/activate
             python ./testpackage/testpackage_get_job_error.py testpackage_run.txt

From 9fe464cac49372241e929db5f8de424914396bee Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Fri, 9 Jan 2026 12:15:00 +0200
Subject: [PATCH 15/28] Github variable did not work, just doing it with grep i
 guess

---
 .github/workflows/test_compare_images.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test_compare_images.yml b/.github/workflows/test_compare_images.yml
index 2d139f0d..8042a850 100644
--- a/.github/workflows/test_compare_images.yml
+++ b/.github/workflows/test_compare_images.yml
@@ -58,10 +58,8 @@ jobs:
             fi
             echo "DIFFRESULT=$DIFFRESULT" >> $GITHUB_OUTPUT
             echo "Running $DIFFRESULT"
-            sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh $DIFFRESULT
-
-            echo ${{ steps.run_cl.outputs.SLURM_JOB_ID }}
-            sacct -j ${{ steps.run_cl.outputs.SLURM_JOB_ID }} -o job,state,node | grep FAILED
+            JOBID=$(sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh $DIFFRESULT | grep -Po '\d+') 
+            sacct -j $JOBID -o job,state,node | grep FAILED
             cat testpackage_run.txt
             . CI_env/bin/activate
             python ./testpackage/testpackage_get_job_error.py testpackage_run.txt

From 1bba6f0e0831b5b23e6bb7ac050900fee4232d25 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Fri, 9 Jan 2026 13:09:29 +0200
Subject: [PATCH 16/28] Added missing export

---
 .github/workflows/test_compare_images.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_compare_images.yml b/.github/workflows/test_compare_images.yml
index 8042a850..6b3b557e 100644
--- a/.github/workflows/test_compare_images.yml
+++ b/.github/workflows/test_compare_images.yml
@@ -58,7 +58,7 @@ jobs:
             fi
             echo "DIFFRESULT=$DIFFRESULT" >> $GITHUB_OUTPUT
             echo "Running $DIFFRESULT"
-            JOBID=$(sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh $DIFFRESULT | grep -Po '\d+') 
+            export JOBID=$(sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh $DIFFRESULT | grep -Po '\d+') 
             sacct -j $JOBID -o job,state,node | grep FAILED
             cat testpackage_run.txt
             . CI_env/bin/activate

From 09fad458462101ec3c9fb9aa201eb61c5afaf543 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Fri, 9 Jan 2026 13:35:56 +0200
Subject: [PATCH 17/28] Third time trying to capture the job id, juts writing
 to a file :/

---
 .github/workflows/test_compare_images.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_compare_images.yml b/.github/workflows/test_compare_images.yml
index 6b3b557e..9550d2b9 100644
--- a/.github/workflows/test_compare_images.yml
+++ b/.github/workflows/test_compare_images.yml
@@ -58,7 +58,8 @@ jobs:
             fi
             echo "DIFFRESULT=$DIFFRESULT" >> $GITHUB_OUTPUT
             echo "Running $DIFFRESULT"
-            export JOBID=$(sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh $DIFFRESULT | grep -Po '\d+') 
+            sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh $DIFFRESULT > jobid.txt
+            export $JOBID=$(grep -Po '\d+' jobid.txt)
             sacct -j $JOBID -o job,state,node | grep FAILED
             cat testpackage_run.txt
             . CI_env/bin/activate

From 7f4f6a783cddf9dec8e1ec2f26b9329cf98d7d61 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Fri, 9 Jan 2026 14:01:48 +0200
Subject: [PATCH 18/28] Final mistake found probably(?)

---
 .github/workflows/test_compare_images.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_compare_images.yml b/.github/workflows/test_compare_images.yml
index 9550d2b9..b457a236 100644
--- a/.github/workflows/test_compare_images.yml
+++ b/.github/workflows/test_compare_images.yml
@@ -59,7 +59,7 @@ jobs:
             echo "DIFFRESULT=$DIFFRESULT" >> $GITHUB_OUTPUT
             echo "Running $DIFFRESULT"
             sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh $DIFFRESULT > jobid.txt
-            export $JOBID=$(grep -Po '\d+' jobid.txt)
+            export JOBID=$(grep -Po '\d+' jobid.txt)
             sacct -j $JOBID -o job,state,node | grep FAILED
             cat testpackage_run.txt
             . CI_env/bin/activate

From 5235847d71b5ff8e9f1063d65d6d107f8922f3f2 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Fri, 9 Jan 2026 14:08:43 +0200
Subject: [PATCH 19/28] Now the workflow should fail if job failed on the node

---
 .github/workflows/test_compare_images.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_compare_images.yml b/.github/workflows/test_compare_images.yml
index b457a236..a3daa822 100644
--- a/.github/workflows/test_compare_images.yml
+++ b/.github/workflows/test_compare_images.yml
@@ -60,7 +60,12 @@ jobs:
             echo "Running $DIFFRESULT"
             sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh $DIFFRESULT > jobid.txt
             export JOBID=$(grep -Po '\d+' jobid.txt)
-            sacct -j $JOBID -o job,state,node | grep FAILED
+            export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED) 
+            if [[ $SACCT_LOG ]]; then
+              echo "Some job failed on a node"
+              echo $SACCT_LOG 
+              exit 1
+            fi
             cat testpackage_run.txt
             . CI_env/bin/activate
             python ./testpackage/testpackage_get_job_error.py testpackage_run.txt

From 6446eb481483e1c9dfe17a269f27891328032f54 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Fri, 9 Jan 2026 16:09:14 +0200
Subject: [PATCH 20/28] Same sacct thing for other workflows

---
 .github/workflows/generate_reference_plots.yml         | 10 ++++++++--
 .github/workflows/test_compare_image_oldest_python.yml | 10 ++++++++--
 .github/workflows/test_compare_images_full.yml         | 10 ++++++++--
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/generate_reference_plots.yml b/.github/workflows/generate_reference_plots.yml
index 633f91c8..d27f0b98 100644
--- a/.github/workflows/generate_reference_plots.yml
+++ b/.github/workflows/generate_reference_plots.yml
@@ -42,8 +42,14 @@ jobs:
             DATAPATH="/wrk-kappa/group/spacephysics/analysator/CI/verification_sets/$(git rev-parse HEAD)"
             mkdir $DATAPATH
             touch $DATAPATH/.lockfile
-            sbatch -W -o testpackage_gen_verf.txt --job-name gen_verf_plots ./testpackage/run_testpackage_generate_verf_set.sh $DATAPATH
-
+            sbatch -W -o testpackage_gen_verf.txt --job-name gen_verf_plots ./testpackage/run_testpackage_generate_verf_set.sh $DATAPATH > jobid.txt
+            export JOBID=$(grep -Po '\d+' jobid.txt)
+            export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED) 
+            if [[ $SACCT_LOG ]]; then
+              echo "Some job failed on a node"
+              echo $SACCT_LOG 
+              exit 1
+            fi
             cat testpackage_gen_verf.txt
 
             python ./testpackage/testpackage_get_job_error.py testpackage_gen_verf.txt
diff --git a/.github/workflows/test_compare_image_oldest_python.yml b/.github/workflows/test_compare_image_oldest_python.yml
index 57517d21..699a2641 100644
--- a/.github/workflows/test_compare_image_oldest_python.yml
+++ b/.github/workflows/test_compare_image_oldest_python.yml
@@ -73,8 +73,14 @@ jobs:
               echo -e ".lockfile found in $verf_loc/$verfset, not running test, as the verification set generation is likely still ongoing\n Check ongoing actions and/or re-run verification set generation."
               exit 1
             fi 
-            sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh
-
+            sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh > jobid.txt
+            export JOBID=$(grep -Po '\d+' jobid.txt)
+            export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED) 
+            if [[ $SACCT_LOG ]]; then
+              echo "Some job failed on a node"
+              echo $SACCT_LOG 
+              exit 1
+            fi
             cat testpackage_run.txt
             . CI_env/bin/activate
             python ./testpackage/testpackage_get_job_error.py testpackage_run.txt
diff --git a/.github/workflows/test_compare_images_full.yml b/.github/workflows/test_compare_images_full.yml
index 1e92dac7..0b0f427f 100644
--- a/.github/workflows/test_compare_images_full.yml
+++ b/.github/workflows/test_compare_images_full.yml
@@ -49,8 +49,14 @@ jobs:
               echo -e ".lockfile found in $verf_loc/$verfset, not running test, as the verification set generation is likely still ongoing\n Check ongoing actions and/or re-run verification set generation."
               exit 1
             fi 
-            sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh
-
+            sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh > jobid.txt
+            export JOBID=$(grep -Po '\d+' jobid.txt)
+            export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED) 
+            if [[ $SACCT_LOG ]]; then
+              echo "Some job failed on a node"
+              echo $SACCT_LOG 
+              exit 1
+            fi
             cat testpackage_run.txt
             . CI_env/bin/activate
             python ./testpackage/testpackage_get_job_error.py testpackage_run.txt

From 4a65c5563759a233ff525555536496cad2860109 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Mon, 12 Jan 2026 14:49:06 +0200
Subject: [PATCH 21/28] Excluded foss pythons since they refuse to work, and
 added head -n1 so we take the first(oldest) if there are multiple versions
 available

---
 .github/workflows/test_compare_image_oldest_python.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_compare_image_oldest_python.yml b/.github/workflows/test_compare_image_oldest_python.yml
index 57517d21..23b6a1ab 100644
--- a/.github/workflows/test_compare_image_oldest_python.yml
+++ b/.github/workflows/test_compare_image_oldest_python.yml
@@ -36,7 +36,7 @@ jobs:
             while [[ ! $VERSIONS_MATCH ]]
             do
               echo "Trying to find module for Python/$PYTHON"
-              VERSIONS_MATCH=$(echo $AVAILABLE_VERSIONS | grep -Po 'Python/.+?-' | grep -Po 'Python'/$PYTHON)
+              VERSIONS_MATCH=$(echo $AVAILABLE_VERSIONS | grep -Po 'Python/.+?-GCC.+' | grep -Po 'Python'/$PYTHON | head -n1)
               if [[ ! $VERSIONS_MATCH ]]; then          
                 MINOR=$(( $MINOR + 1 ))
                 PYTHON=$MAJOR.$MINOR

From 5bcaaf8dfb7e4db44a7a9ff1ca6da9871bd80636 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Mon, 12 Jan 2026 14:51:10 +0200
Subject: [PATCH 22/28] added module list to the oldest python workflow also
 module purge to a substep for good measure

---
 .github/workflows/test_compare_image_oldest_python.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test_compare_image_oldest_python.yml b/.github/workflows/test_compare_image_oldest_python.yml
index 23b6a1ab..71ea3e9d 100644
--- a/.github/workflows/test_compare_image_oldest_python.yml
+++ b/.github/workflows/test_compare_image_oldest_python.yml
@@ -67,6 +67,7 @@ jobs:
             export TMPDIR=$RUNNER_TEMP
             module purge
             module load Python/${{ steps.pyversion.outputs.PYTHON }}
+            module list
             verf_loc="/wrk-kappa/group/spacephysics/analysator/CI/verification_sets"
             verfset=$(ls -lth $verf_loc | grep ^d | head -n1 | grep -Po '\w+$')
             if [[ -f $verf_loc/$verfset/.lockfile ]]; then 
@@ -82,7 +83,9 @@ jobs:
         - name: Comparing plotted data
           run: |
             export TMPDIR=$RUNNER_TEMP
+            module purge
             module load Python/${{ steps.pyversion.outputs.PYTHON }}
+            module list
             . CI_env/bin/activate
             sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh
             cat testpackage_compare.txt

From 97c1caa709d71d60656a15478c72452087ed38cb Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Mon, 12 Jan 2026 15:18:59 +0200
Subject: [PATCH 23/28] Fixing the version matching so it takes the oldest
 found major.minor  GCCcore python

---
 .github/workflows/test_compare_image_oldest_python.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_compare_image_oldest_python.yml b/.github/workflows/test_compare_image_oldest_python.yml
index 571e4af9..32df2faa 100644
--- a/.github/workflows/test_compare_image_oldest_python.yml
+++ b/.github/workflows/test_compare_image_oldest_python.yml
@@ -36,7 +36,7 @@ jobs:
             while [[ ! $VERSIONS_MATCH ]]
             do
               echo "Trying to find module for Python/$PYTHON"
-              VERSIONS_MATCH=$(echo $AVAILABLE_VERSIONS | grep -Po 'Python/.+?-GCC.+' | grep -Po 'Python'/$PYTHON | head -n1)
+              VERSIONS_MATCH=$(echo "$AVAILABLE_VERSIONS" | grep -Po 'Python/.+?-GCC.+' | grep -P 'Python'/$PYTHON | head -n1)
               if [[ ! $VERSIONS_MATCH ]]; then          
                 MINOR=$(( $MINOR + 1 ))
                 PYTHON=$MAJOR.$MINOR
@@ -47,8 +47,9 @@ jobs:
                 fi
               fi
             done
-            echo "PYTHON=$PYTHON" >> $GITHUB_OUTPUT
-            module load Python/$PYTHON
+            echo "Found $VERSIONS_MATCH"
+            echo "PYTHON=$VERSIONS_MATCH" >> $GITHUB_OUTPUT
+            module load $VERSIONS_MATCH
             module list Python
         - name: Install dependencies
           timeout-minutes: 5

From 096bf4b0bc7207b83e3f75ce7bd7410afb89b3d8 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Mon, 12 Jan 2026 15:26:20 +0200
Subject: [PATCH 24/28] Fixing the old module loads to use the new format,
 forgot

---
 .github/workflows/test_compare_image_oldest_python.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_compare_image_oldest_python.yml b/.github/workflows/test_compare_image_oldest_python.yml
index 32df2faa..3efe7cd2 100644
--- a/.github/workflows/test_compare_image_oldest_python.yml
+++ b/.github/workflows/test_compare_image_oldest_python.yml
@@ -57,7 +57,7 @@ jobs:
             export TMPDIR=$RUNNER_TEMP
             export UV_LINK_MODE=copy
             module purge
-            module load Python/${{ steps.pyversion.outputs.PYTHON }}
+            module load ${{ steps.pyversion.outputs.PYTHON }}
             module list
             uv venv CI_env
             . CI_env/bin/activate
@@ -67,7 +67,7 @@ jobs:
           run: |
             export TMPDIR=$RUNNER_TEMP
             module purge
-            module load Python/${{ steps.pyversion.outputs.PYTHON }}
+            module load ${{ steps.pyversion.outputs.PYTHON }}
             module list
             verf_loc="/wrk-kappa/group/spacephysics/analysator/CI/verification_sets"
             verfset=$(ls -lth $verf_loc | grep ^d | head -n1 | grep -Po '\w+$')
@@ -91,7 +91,7 @@ jobs:
           run: |
             export TMPDIR=$RUNNER_TEMP
             module purge
-            module load Python/${{ steps.pyversion.outputs.PYTHON }}
+            module load ${{ steps.pyversion.outputs.PYTHON }}
             module list
             . CI_env/bin/activate
             sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh

From eb5b9608bf1f993632f487a29d00695b8f573278 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Mon, 12 Jan 2026 15:31:43 +0200
Subject: [PATCH 25/28] Added warning for the python versions so it is more
 easily found in the github logs

---
 .github/workflows/test_compare_image_oldest_python.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_compare_image_oldest_python.yml b/.github/workflows/test_compare_image_oldest_python.yml
index 3efe7cd2..8d28abac 100644
--- a/.github/workflows/test_compare_image_oldest_python.yml
+++ b/.github/workflows/test_compare_image_oldest_python.yml
@@ -47,7 +47,7 @@ jobs:
                 fi
               fi
             done
-            echo "Found $VERSIONS_MATCH"
+            echo "::warning::Found $VERSIONS_MATCH"
             echo "PYTHON=$VERSIONS_MATCH" >> $GITHUB_OUTPUT
             module load $VERSIONS_MATCH
             module list Python

From 0d10890ea35de6ca4a06872f0c5d739b9cbfb861 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Mon, 12 Jan 2026 15:43:05 +0200
Subject: [PATCH 26/28] Added a way to feed the python version to
 run_testpackage_workflow by calling it with old_python [Full module name for
 python]

---
 .../test_compare_image_oldest_python.yml         |  2 +-
 testpackage/run_testpackage_workflow.sh          | 16 +++++++++++++---
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test_compare_image_oldest_python.yml b/.github/workflows/test_compare_image_oldest_python.yml
index 8d28abac..e7b1ab1b 100644
--- a/.github/workflows/test_compare_image_oldest_python.yml
+++ b/.github/workflows/test_compare_image_oldest_python.yml
@@ -75,7 +75,7 @@ jobs:
               echo -e ".lockfile found in $verf_loc/$verfset, not running test, as the verification set generation is likely still ongoing\n Check ongoing actions and/or re-run verification set generation."
               exit 1
             fi 
-            sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh > jobid.txt
+            sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh old_python ${{ steps.pyversion.outputs.PYTHON }} > jobid.txt
             export JOBID=$(grep -Po '\d+' jobid.txt)
             export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED) 
             if [[ $SACCT_LOG ]]; then
diff --git a/testpackage/run_testpackage_workflow.sh b/testpackage/run_testpackage_workflow.sh
index 9eaa6e96..6e1e9a30 100755
--- a/testpackage/run_testpackage_workflow.sh
+++ b/testpackage/run_testpackage_workflow.sh
@@ -19,9 +19,14 @@ hostname
 
 module purge            
 export PATH=/wrk-vakka/group/spacephysics/proj/appl/tex-basic/texlive/2023/bin/x86_64-linux:$PATH
-
+older_python=false
 echo "SLURM_JOB_ID=$SLURM_ARRAY_JOB_ID" >> $GITHUB_OUTPUT
-module load Python/3.10.4-GCCcore-11.3.0
+if [[ $1 == "old_python" ]]; then
+  module load $2
+  older_python=true
+else
+  module load Python/3.10.4-GCCcore-11.3.0
+fi
 source CI_env/bin/activate
 
 module list
@@ -32,6 +37,11 @@ export PTNONINTERACTIVE=1
 export PTOUTPUTDIR=$PWD/produced_plots/
 
 
-
+if $older_python; then
+  echo "::warning:: Running with older python version $2"
+  python ./testpackage/testpackage_commons $jobcount $index 
+  echo "EXIT_CODE_FROM_JOB $?"
+  exit 0
+fi
 python ./testpackage/testpackage_commons.py $jobcount $index $@
 echo "EXIT_CODE_FROM_JOB $?"

From 1d6879d5fbca1dbd662888cf01487c6cb49d9ea7 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Mon, 12 Jan 2026 15:50:39 +0200
Subject: [PATCH 27/28] typo fixed

---
 testpackage/run_testpackage_workflow.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testpackage/run_testpackage_workflow.sh b/testpackage/run_testpackage_workflow.sh
index 6e1e9a30..d401be0f 100755
--- a/testpackage/run_testpackage_workflow.sh
+++ b/testpackage/run_testpackage_workflow.sh
@@ -39,7 +39,7 @@ export PTOUTPUTDIR=$PWD/produced_plots/
 
 if $older_python; then
   echo "::warning:: Running with older python version $2"
-  python ./testpackage/testpackage_commons $jobcount $index 
+  python ./testpackage/testpackage_commons.py $jobcount $index 
   echo "EXIT_CODE_FROM_JOB $?"
   exit 0
 fi

From 611bb7d71ba6244d583b19facdc221268dff7f32 Mon Sep 17 00:00:00 2001
From: lassjsc <lasse.b.sihvonen@helsinki.fi>
Date: Mon, 12 Jan 2026 16:22:18 +0200
Subject: [PATCH 28/28] Older python load option for run_compare also

---
 .github/workflows/test_compare_image_oldest_python.yml | 2 +-
 testpackage/run_compare.sh                             | 9 ++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_compare_image_oldest_python.yml b/.github/workflows/test_compare_image_oldest_python.yml
index e7b1ab1b..b45a82c7 100644
--- a/.github/workflows/test_compare_image_oldest_python.yml
+++ b/.github/workflows/test_compare_image_oldest_python.yml
@@ -94,7 +94,7 @@ jobs:
             module load ${{ steps.pyversion.outputs.PYTHON }}
             module list
             . CI_env/bin/activate
-            sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh
+            sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh old_python ${{ steps.pyversion.outputs.PYTHON }}
             cat testpackage_compare.txt
             python ./testpackage/testpackage_get_job_error.py testpackage_compare.txt
         - name: scancel dangling job upon cancellation
diff --git a/testpackage/run_compare.sh b/testpackage/run_compare.sh
index 24ce3d71..981445fe 100755
--- a/testpackage/run_compare.sh
+++ b/testpackage/run_compare.sh
@@ -16,7 +16,11 @@ index=$(( $SLURM_ARRAY_TASK_ID - $SLURM_ARRAY_TASK_MIN ))
 hostname
 
 module purge
-module load Python/3.10.4-GCCcore-11.3.0
+if [[ $1 == 'old_python' ]]; then
+  module load $2
+else
+  module load Python/3.10.4-GCCcore-11.3.0
+fi
 source CI_env/bin/activate
 module load libglvnd/1.7.0-GCCcore-13.3.0
 module list
@@ -48,6 +52,9 @@ if [[ $@ == 'verf_set' ]]; then
   python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index 0 && echo "No differences found" 
   echo "EXIT_CODE_FROM_JOB $?"
   exit 0
+#If old_python is used, run all
+elif [[ $1 == 'old_python' ]]; then
+  check=false
 #Do selective compare if other arguments
 elif [ $@ ]; then
   check=true