From ecd4685ececd028e1cc9792c6a10eb6b6e870819 Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 18:52:45 -0600
Subject: [PATCH 01/20] Add action

---
 .github/workflows/daily-update.yml | 102 +++++++++++++++++++++++++++++
 .github/workflows/go.yml           |   2 +-
 2 files changed, 103 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/daily-update.yml

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
new file mode 100644
index 0000000..abe166d
--- /dev/null
+++ b/.github/workflows/daily-update.yml
@@ -0,0 +1,102 @@
+# This workflow scrapes, parses, and uploads data to MongoDB on a daily basis
+
+name: Go
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '10 1 * * *'
+
+jobs:
+
+  astra:
+    runs-on: ubuntu-latest
+    steps:
+    
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.23'
+
+    - name: Fill env for dev
+      run: |
+        cp .env.example .env
+        sed -i 's|LOGIN_ASTRA_USERNAME=|LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
+        sed -i 's|LOGIN_ASTRA_PASSWORD=/LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}|' .env
+        sed -i 's|MAZEVO_API_KEY=/MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}|' .env
+        sed -i 's|MONGODB_URI=/MONGODB_URI=${{ secrets.MONGODB_URI_DEV }}|' .env
+
+    - name: Build
+      run: make
+
+    - name: Scrape Astra
+      run: api-tools.exe -scrape -astra
+
+#    - name: Parse Astra
+#      run: api-tools.exe -parse -astra
+
+#    - name: Upload Astra
+#      run: api-tools.exe -upload -events
+
+    - name: Fill env for prod
+      run: |
+        cp .env.example .env
+        sed -i 's|LOGIN_ASTRA_USERNAME=|LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
+        sed -i 's|LOGIN_ASTRA_PASSWORD=/LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}|' .env
+        sed -i 's|MAZEVO_API_KEY=/MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}|' .env
+        sed -i 's|MONGODB_URI=/MONGODB_URI=${{ secrets.MONGODB_URI_PROD }}|' .env
+
+    - name: Build
+      run: make
+
+#    - name: Upload Astra
+#      run: api-tools.exe -upload -events
+
+  mazevo:
+    runs-on: ubuntu-latest
+    steps:
+    
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.23'
+
+    - name: Fill env for dev
+      run: |
+        cp .env.example .env
+        sed -i 's|LOGIN_ASTRA_USERNAME=|LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
+        sed -i 's|LOGIN_ASTRA_PASSWORD=/LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}|' .env
+        sed -i 's|MAZEVO_API_KEY=/MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}|' .env
+        sed -i 's|MONGODB_URI=/MONGODB_URI=${{ secrets.MONGODB_URI_DEV }}|' .env
+
+    - name: Build
+      run: make
+
+    - name: Scrape Mazevo
+      run: api-tools.exe -scrape -mazevo
+
+#    - name: Parse Mazevo
+#      run: api-tools.exe -parse -mazevo
+
+#    - name: Upload Mazevo
+#      run: api-tools.exe -upload -events
+
+    - name: Fill env for prod
+      run: |
+        cp .env.example .env
+        sed -i 's|LOGIN_ASTRA_USERNAME=|LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
+        sed -i 's|LOGIN_ASTRA_PASSWORD=/LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}|' .env
+        sed -i 's|MAZEVO_API_KEY=/MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}|' .env
+        sed -i 's|MONGODB_URI=/MONGODB_URI=${{ secrets.MONGODB_URI_PROD }}|' .env
+
+    - name: Build
+      run: make
+
+#    - name: Upload Mazevo
+#      run: api-tools.exe -upload -events
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index 6422e42..3629edc 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -17,7 +17,7 @@ jobs:
     steps:
     
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Go
       uses: actions/setup-go@v4

From 0a2d8dee13a95298f01a479a77ddaca854777204 Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 19:09:25 -0600
Subject: [PATCH 02/20] A tool online told me I should

---
 .github/workflows/daily-update.yml | 4 ++--
 .github/workflows/go.yml           | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
index abe166d..a3dbbca 100644
--- a/.github/workflows/daily-update.yml
+++ b/.github/workflows/daily-update.yml
@@ -17,7 +17,7 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: '1.23'
 
@@ -63,7 +63,7 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: '1.23'
 
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index 3629edc..ca57a2b 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -20,7 +20,7 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Set up Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version: '1.23'
 

From e1ccca1f4c8e87beb727cb171cd6982476f22b45 Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 19:09:37 -0600
Subject: [PATCH 03/20] To show up in the actions tab

---
 .github/workflows/daily-update.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
index a3dbbca..7c8fd7f 100644
--- a/.github/workflows/daily-update.yml
+++ b/.github/workflows/daily-update.yml
@@ -6,6 +6,7 @@ on:
   workflow_dispatch:
   schedule:
     - cron: '10 1 * * *'
+  pull_request: # temporary until merged
 
 jobs:
 

From 75c66c0d55f8d231972e0289156b05d9fd779fa3 Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 19:16:14 -0600
Subject: [PATCH 04/20] .env.example -> .env.template

---
 .github/workflows/daily-update.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
index 7c8fd7f..17a45ed 100644
--- a/.github/workflows/daily-update.yml
+++ b/.github/workflows/daily-update.yml
@@ -1,6 +1,6 @@
 # This workflow scrapes, parses, and uploads data to MongoDB on a daily basis
 
-name: Go
+name: Daily update
 
 on:
   workflow_dispatch:
@@ -24,7 +24,7 @@ jobs:
 
     - name: Fill env for dev
       run: |
-        cp .env.example .env
+        cp .env.template .env
         sed -i 's|LOGIN_ASTRA_USERNAME=|LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
         sed -i 's|LOGIN_ASTRA_PASSWORD=/LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}|' .env
         sed -i 's|MAZEVO_API_KEY=/MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}|' .env
@@ -44,7 +44,7 @@ jobs:
 
     - name: Fill env for prod
       run: |
-        cp .env.example .env
+        cp .env.template .env
         sed -i 's|LOGIN_ASTRA_USERNAME=|LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
         sed -i 's|LOGIN_ASTRA_PASSWORD=/LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}|' .env
         sed -i 's|MAZEVO_API_KEY=/MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}|' .env
@@ -70,7 +70,7 @@ jobs:
 
     - name: Fill env for dev
       run: |
-        cp .env.example .env
+        cp .env.template .env
         sed -i 's|LOGIN_ASTRA_USERNAME=|LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
         sed -i 's|LOGIN_ASTRA_PASSWORD=/LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}|' .env
         sed -i 's|MAZEVO_API_KEY=/MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}|' .env
@@ -90,7 +90,7 @@ jobs:
 
     - name: Fill env for prod
       run: |
-        cp .env.example .env
+        cp .env.template .env
         sed -i 's|LOGIN_ASTRA_USERNAME=|LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
         sed -i 's|LOGIN_ASTRA_PASSWORD=/LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}|' .env
         sed -i 's|MAZEVO_API_KEY=/MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}|' .env

From b144762c12fc978578ba26c51dd160f2443a9f7e Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 19:28:15 -0600
Subject: [PATCH 05/20] sed character change: | -> /

---
 .github/workflows/daily-update.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
index 17a45ed..4f69407 100644
--- a/.github/workflows/daily-update.yml
+++ b/.github/workflows/daily-update.yml
@@ -25,7 +25,7 @@ jobs:
     - name: Fill env for dev
       run: |
         cp .env.template .env
-        sed -i 's|LOGIN_ASTRA_USERNAME=|LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
+        sed -i 's|LOGIN_ASTRA_USERNAME=/LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
         sed -i 's|LOGIN_ASTRA_PASSWORD=/LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}|' .env
         sed -i 's|MAZEVO_API_KEY=/MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}|' .env
         sed -i 's|MONGODB_URI=/MONGODB_URI=${{ secrets.MONGODB_URI_DEV }}|' .env
@@ -45,7 +45,7 @@ jobs:
     - name: Fill env for prod
       run: |
         cp .env.template .env
-        sed -i 's|LOGIN_ASTRA_USERNAME=|LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
+        sed -i 's|LOGIN_ASTRA_USERNAME=/LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
         sed -i 's|LOGIN_ASTRA_PASSWORD=/LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}|' .env
         sed -i 's|MAZEVO_API_KEY=/MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}|' .env
         sed -i 's|MONGODB_URI=/MONGODB_URI=${{ secrets.MONGODB_URI_PROD }}|' .env
@@ -71,7 +71,7 @@ jobs:
     - name: Fill env for dev
       run: |
         cp .env.template .env
-        sed -i 's|LOGIN_ASTRA_USERNAME=|LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
+        sed -i 's|LOGIN_ASTRA_USERNAME=/LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
         sed -i 's|LOGIN_ASTRA_PASSWORD=/LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}|' .env
         sed -i 's|MAZEVO_API_KEY=/MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}|' .env
         sed -i 's|MONGODB_URI=/MONGODB_URI=${{ secrets.MONGODB_URI_DEV }}|' .env
@@ -91,7 +91,7 @@ jobs:
     - name: Fill env for prod
       run: |
         cp .env.template .env
-        sed -i 's|LOGIN_ASTRA_USERNAME=|LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
+        sed -i 's|LOGIN_ASTRA_USERNAME=/LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
         sed -i 's|LOGIN_ASTRA_PASSWORD=/LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}|' .env
         sed -i 's|MAZEVO_API_KEY=/MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}|' .env
         sed -i 's|MONGODB_URI=/MONGODB_URI=${{ secrets.MONGODB_URI_PROD }}|' .env

From f8332f5fbc58cf022fcafe007044a0d2ab5d43ef Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 19:44:01 -0600
Subject: [PATCH 06/20] Change sed delimeter

---
 .github/workflows/daily-update.yml | 32 +++++++++++++++---------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
index 4f69407..923ea7a 100644
--- a/.github/workflows/daily-update.yml
+++ b/.github/workflows/daily-update.yml
@@ -25,10 +25,10 @@ jobs:
     - name: Fill env for dev
       run: |
         cp .env.template .env
-        sed -i 's|LOGIN_ASTRA_USERNAME=/LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
-        sed -i 's|LOGIN_ASTRA_PASSWORD=/LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}|' .env
-        sed -i 's|MAZEVO_API_KEY=/MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}|' .env
-        sed -i 's|MONGODB_URI=/MONGODB_URI=${{ secrets.MONGODB_URI_DEV }}|' .env
+        sed -i 's#LOGIN_ASTRA_USERNAME=#LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}#' .env
+        sed -i 's#LOGIN_ASTRA_PASSWORD=#LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}#' .env
+        sed -i 's#MAZEVO_API_KEY=#MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}#' .env
+        sed -i 's#MONGODB_URI=#MONGODB_URI=${{ secrets.MONGODB_URI_DEV }}#' .env
 
     - name: Build
       run: make
@@ -45,10 +45,10 @@ jobs:
     - name: Fill env for prod
       run: |
         cp .env.template .env
-        sed -i 's|LOGIN_ASTRA_USERNAME=/LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
-        sed -i 's|LOGIN_ASTRA_PASSWORD=/LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}|' .env
-        sed -i 's|MAZEVO_API_KEY=/MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}|' .env
-        sed -i 's|MONGODB_URI=/MONGODB_URI=${{ secrets.MONGODB_URI_PROD }}|' .env
+        sed -i 's#LOGIN_ASTRA_USERNAME=#LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}#' .env
+        sed -i 's#LOGIN_ASTRA_PASSWORD=#LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}#' .env
+        sed -i 's#MAZEVO_API_KEY=#MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}#' .env
+        sed -i 's#MONGODB_URI=#MONGODB_URI=${{ secrets.MONGODB_URI_PROD }}#' .env
 
     - name: Build
       run: make
@@ -71,10 +71,10 @@ jobs:
     - name: Fill env for dev
       run: |
         cp .env.template .env
-        sed -i 's|LOGIN_ASTRA_USERNAME=/LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
-        sed -i 's|LOGIN_ASTRA_PASSWORD=/LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}|' .env
-        sed -i 's|MAZEVO_API_KEY=/MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}|' .env
-        sed -i 's|MONGODB_URI=/MONGODB_URI=${{ secrets.MONGODB_URI_DEV }}|' .env
+        sed -i 's#LOGIN_ASTRA_USERNAME=#LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}#' .env
+        sed -i 's#LOGIN_ASTRA_PASSWORD=#LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}#' .env
+        sed -i 's#MAZEVO_API_KEY=#MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}#' .env
+        sed -i 's#MONGODB_URI=#MONGODB_URI=${{ secrets.MONGODB_URI_DEV }}#' .env
 
     - name: Build
       run: make
@@ -91,10 +91,10 @@ jobs:
     - name: Fill env for prod
       run: |
         cp .env.template .env
-        sed -i 's|LOGIN_ASTRA_USERNAME=/LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}|' .env
-        sed -i 's|LOGIN_ASTRA_PASSWORD=/LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}|' .env
-        sed -i 's|MAZEVO_API_KEY=/MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}|' .env
-        sed -i 's|MONGODB_URI=/MONGODB_URI=${{ secrets.MONGODB_URI_PROD }}|' .env
+        sed -i 's#LOGIN_ASTRA_USERNAME=#LOGIN_ASTRA_USERNAME=${{ secrets.LOGIN_ASTRA_USERNAME }}#' .env
+        sed -i 's#LOGIN_ASTRA_PASSWORD=#LOGIN_ASTRA_PASSWORD=${{ secrets.LOGIN_ASTRA_PASSWORD }}#' .env
+        sed -i 's#MAZEVO_API_KEY=#MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}#' .env
+        sed -i 's#MONGODB_URI=#MONGODB_URI=${{ secrets.MONGODB_URI_PROD }}#' .env
 
     - name: Build
       run: make

From 08e6b02208ac915124110eb9ae0e55a6a4785449 Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 19:45:32 -0600
Subject: [PATCH 07/20] Fix .exe run command

---
 .github/workflows/daily-update.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
index 923ea7a..64f1d28 100644
--- a/.github/workflows/daily-update.yml
+++ b/.github/workflows/daily-update.yml
@@ -34,13 +34,13 @@ jobs:
       run: make
 
     - name: Scrape Astra
-      run: api-tools.exe -scrape -astra
+      run: ./api-tools.exe -scrape -astra
 
 #    - name: Parse Astra
-#      run: api-tools.exe -parse -astra
+#      run: ./api-tools.exe -parse -astra
 
 #    - name: Upload Astra
-#      run: api-tools.exe -upload -events
+#      run: ./api-tools.exe -upload -events
 
     - name: Fill env for prod
       run: |
@@ -54,7 +54,7 @@ jobs:
       run: make
 
 #    - name: Upload Astra
-#      run: api-tools.exe -upload -events
+#      run: ./api-tools.exe -upload -events
 
   mazevo:
     runs-on: ubuntu-latest
@@ -80,13 +80,13 @@ jobs:
       run: make
 
     - name: Scrape Mazevo
-      run: api-tools.exe -scrape -mazevo
+      run: ./api-tools.exe -scrape -mazevo
 
 #    - name: Parse Mazevo
-#      run: api-tools.exe -parse -mazevo
+#      run: ./api-tools.exe -parse -mazevo
 
 #    - name: Upload Mazevo
-#      run: api-tools.exe -upload -events
+#      run: ./api-tools.exe -upload -events
 
     - name: Fill env for prod
       run: |
@@ -100,4 +100,4 @@ jobs:
       run: make
 
 #    - name: Upload Mazevo
-#      run: api-tools.exe -upload -events
+#      run: ./api-tools.exe -upload -events

From 49cfb94c2315260aacfd6493ee6f06bd4e352667 Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 19:50:37 -0600
Subject: [PATCH 08/20] Full build

---
 .github/workflows/daily-update.yml | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
index 64f1d28..e051664 100644
--- a/.github/workflows/daily-update.yml
+++ b/.github/workflows/daily-update.yml
@@ -30,8 +30,14 @@ jobs:
         sed -i 's#MAZEVO_API_KEY=#MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}#' .env
         sed -i 's#MONGODB_URI=#MONGODB_URI=${{ secrets.MONGODB_URI_DEV }}#' .env
 
+    - name: Setup
+      run: make setup
+
+    - name: Check
+      run: make check
+
     - name: Build
-      run: make
+      run: make build
 
     - name: Scrape Astra
       run: ./api-tools.exe -scrape -astra
@@ -50,9 +56,6 @@ jobs:
         sed -i 's#MAZEVO_API_KEY=#MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}#' .env
         sed -i 's#MONGODB_URI=#MONGODB_URI=${{ secrets.MONGODB_URI_PROD }}#' .env
 
-    - name: Build
-      run: make
-
 #    - name: Upload Astra
 #      run: ./api-tools.exe -upload -events
 
@@ -76,8 +79,14 @@ jobs:
         sed -i 's#MAZEVO_API_KEY=#MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}#' .env
         sed -i 's#MONGODB_URI=#MONGODB_URI=${{ secrets.MONGODB_URI_DEV }}#' .env
 
+    - name: Setup
+      run: make setup
+
+    - name: Check
+      run: make check
+
     - name: Build
-      run: make
+      run: make build
 
     - name: Scrape Mazevo
       run: ./api-tools.exe -scrape -mazevo
@@ -96,8 +105,5 @@ jobs:
         sed -i 's#MAZEVO_API_KEY=#MAZEVO_API_KEY=${{ secrets.MAZEVO_API_KEY }}#' .env
         sed -i 's#MONGODB_URI=#MONGODB_URI=${{ secrets.MONGODB_URI_PROD }}#' .env
 
-    - name: Build
-      run: make
-
 #    - name: Upload Mazevo
 #      run: ./api-tools.exe -upload -events

From 2f603a8fa1f923b151c87f150ea5d895a962bfd3 Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 20:01:29 -0600
Subject: [PATCH 09/20] ls test

---
 .github/workflows/daily-update.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
index e051664..bd59613 100644
--- a/.github/workflows/daily-update.yml
+++ b/.github/workflows/daily-update.yml
@@ -39,6 +39,9 @@ jobs:
     - name: Build
       run: make build
 
+    - name: ls
+      run: ls -a
+
     - name: Scrape Astra
       run: ./api-tools.exe -scrape -astra
 

From c9898033cf656eb209d44beb7e8cd6aac815270e Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 20:04:08 -0600
Subject: [PATCH 10/20] Revert "ls test"

This reverts commit 2f603a8fa1f923b151c87f150ea5d895a962bfd3.
---
 .github/workflows/daily-update.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
index bd59613..e051664 100644
--- a/.github/workflows/daily-update.yml
+++ b/.github/workflows/daily-update.yml
@@ -39,9 +39,6 @@ jobs:
     - name: Build
       run: make build
 
-    - name: ls
-      run: ls -a
-
     - name: Scrape Astra
       run: ./api-tools.exe -scrape -astra
 

From 5ca84613ea086dc0662a9c259854a9b74e525387 Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 20:04:19 -0600
Subject: [PATCH 11/20] api-tools.exe -> api-tools

---
 .github/workflows/daily-update.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
index e051664..54d81b2 100644
--- a/.github/workflows/daily-update.yml
+++ b/.github/workflows/daily-update.yml
@@ -40,13 +40,13 @@ jobs:
       run: make build
 
     - name: Scrape Astra
-      run: ./api-tools.exe -scrape -astra
+      run: ./api-tools -scrape -astra
 
 #    - name: Parse Astra
-#      run: ./api-tools.exe -parse -astra
+#      run: ./api-tools -parse -astra
 
 #    - name: Upload Astra
-#      run: ./api-tools.exe -upload -events
+#      run: ./api-tools -upload -events
 
     - name: Fill env for prod
       run: |
@@ -57,7 +57,7 @@ jobs:
         sed -i 's#MONGODB_URI=#MONGODB_URI=${{ secrets.MONGODB_URI_PROD }}#' .env
 
 #    - name: Upload Astra
-#      run: ./api-tools.exe -upload -events
+#      run: ./api-tools -upload -events
 
   mazevo:
     runs-on: ubuntu-latest
@@ -89,13 +89,13 @@ jobs:
       run: make build
 
     - name: Scrape Mazevo
-      run: ./api-tools.exe -scrape -mazevo
+      run: ./api-tools -scrape -mazevo
 
 #    - name: Parse Mazevo
-#      run: ./api-tools.exe -parse -mazevo
+#      run: ./api-tools -parse -mazevo
 
 #    - name: Upload Mazevo
-#      run: ./api-tools.exe -upload -events
+#      run: ./api-tools -upload -events
 
     - name: Fill env for prod
       run: |
@@ -106,4 +106,4 @@ jobs:
         sed -i 's#MONGODB_URI=#MONGODB_URI=${{ secrets.MONGODB_URI_PROD }}#' .env
 
 #    - name: Upload Mazevo
-#      run: ./api-tools.exe -upload -events
+#      run: ./api-tools -upload -events

From 4167864ac97cd22d448fb8d27b6d34122af51366 Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 20:22:40 -0600
Subject: [PATCH 12/20] Use container with chrome

---
 .github/workflows/daily-update.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
index 54d81b2..e4487d6 100644
--- a/.github/workflows/daily-update.yml
+++ b/.github/workflows/daily-update.yml
@@ -12,6 +12,7 @@ jobs:
 
   astra:
     runs-on: ubuntu-latest
+    container: docker.io/chromedp/headless-shell:latest # container with chrome
     steps:
     
     - name: Checkout

From 990ee7355c1b8d14f3af9d0243ed13324d2eedc9 Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 20:28:29 -0600
Subject: [PATCH 13/20] Install chromium differently

---
 .github/workflows/daily-update.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
index e4487d6..d9d54f3 100644
--- a/.github/workflows/daily-update.yml
+++ b/.github/workflows/daily-update.yml
@@ -12,9 +12,11 @@ jobs:
 
   astra:
     runs-on: ubuntu-latest
-    container: docker.io/chromedp/headless-shell:latest # container with chrome
     steps:
-    
+
+    - name: Install Chromium
+      run: apt-get update && apt-get install -y chromium
+
     - name: Checkout
       uses: actions/checkout@v4
 

From ec692e53f015a930a70a54092337f297a85d7b47 Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 20:30:52 -0600
Subject: [PATCH 14/20] Install chromium differently

---
 .github/workflows/daily-update.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
index d9d54f3..52868d7 100644
--- a/.github/workflows/daily-update.yml
+++ b/.github/workflows/daily-update.yml
@@ -15,7 +15,7 @@ jobs:
     steps:
 
     - name: Install Chromium
-      run: apt-get update && apt-get install -y chromium
+      run: sudo apt-get install -y chromium-browser
 
     - name: Checkout
       uses: actions/checkout@v4

From 6e8db4464a2f27ef21d9d9b49a43bfda86c1ccd5 Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 23:27:33 -0600
Subject: [PATCH 15/20] Install chrome and scrape with xvfb-run

---
 .github/workflows/daily-update.yml | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
index 52868d7..ffb2848 100644
--- a/.github/workflows/daily-update.yml
+++ b/.github/workflows/daily-update.yml
@@ -14,8 +14,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Install Chromium
-      run: sudo apt-get install -y chromium-browser
+    - name: Install Chrome
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y wget gnupg
+        wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
+        sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list'
+        sudo apt-get update
+        sudo apt-get install -y google-chrome-stable
 
     - name: Checkout
       uses: actions/checkout@v4
@@ -43,7 +49,10 @@ jobs:
       run: make build
 
     - name: Scrape Astra
-      run: ./api-tools -scrape -astra
+      env:
+        CHROME_BIN: /usr/bin/google-chrome
+      run: |
+        xvfb-run --auto-servernum ./api-tools -scrape -astra
 
 #    - name: Parse Astra
 #      run: ./api-tools -parse -astra

From 3dcf078cda88f54454e2d80d7f517cc8dc5233c7 Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 23:31:37 -0600
Subject: [PATCH 16/20] Revert "Install chrome and scrape with xvfb-run"

This reverts commit 6e8db4464a2f27ef21d9d9b49a43bfda86c1ccd5.
---
 .github/workflows/daily-update.yml | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
index ffb2848..52868d7 100644
--- a/.github/workflows/daily-update.yml
+++ b/.github/workflows/daily-update.yml
@@ -14,14 +14,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Install Chrome
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y wget gnupg
-        wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
-        sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list'
-        sudo apt-get update
-        sudo apt-get install -y google-chrome-stable
+    - name: Install Chromium
+      run: sudo apt-get install -y chromium-browser
 
     - name: Checkout
       uses: actions/checkout@v4
@@ -49,10 +43,7 @@ jobs:
       run: make build
 
     - name: Scrape Astra
-      env:
-        CHROME_BIN: /usr/bin/google-chrome
-      run: |
-        xvfb-run --auto-servernum ./api-tools -scrape -astra
+      run: ./api-tools -scrape -astra
 
 #    - name: Parse Astra
 #      run: ./api-tools -parse -astra

From 2e4b4b13cfc7a7ddfff579b40a71996f54687fe2 Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 23:35:35 -0600
Subject: [PATCH 17/20] Print statements galore

---
 .github/workflows/daily-update.yml | 2 +-
 scrapers/astra.go                  | 6 ++++++
 utils/methods.go                   | 4 ++++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml
index 52868d7..580bb01 100644
--- a/.github/workflows/daily-update.yml
+++ b/.github/workflows/daily-update.yml
@@ -43,7 +43,7 @@ jobs:
       run: make build
 
     - name: Scrape Astra
-      run: ./api-tools -scrape -astra
+      run: ./api-tools -scrape -astra -verbose
 
 #    - name: Parse Astra
 #      run: ./api-tools -parse -astra
diff --git a/scrapers/astra.go b/scrapers/astra.go
index d2cf5b1..b00730e 100644
--- a/scrapers/astra.go
+++ b/scrapers/astra.go
@@ -26,17 +26,21 @@ func ScrapeAstra(outDir string) {
 		log.Panic("Error loading .env file")
 	}
 
+	log.Print("1")
 	// Start chromedp
 	chromedpCtx, cancel := utils.InitChromeDp()
+	log.Print("2")
 
 	// Make output folder
 	err := os.MkdirAll(outDir, 0777)
 	if err != nil {
 		panic(err)
 	}
+	log.Print("3")
 
 	days := "{"       // String JSON for storing results by day
 	firstLoop := true // To avoid adding a comma to the JSON on the first loop
+	log.Print("4")
 
 	// Init http client
 	tr := &http.Transport{
@@ -45,11 +49,13 @@ func ScrapeAstra(outDir string) {
 		DisableCompression: true,
 	}
 	cli := &http.Client{Transport: tr}
+	log.Print("5")
 
 	// Get cookies for auth
 	astraHeaders := utils.RefreshAstraToken(chromedpCtx)
 	time.Sleep(500 * time.Millisecond)
 	cancel() // Don't need chromedp anymore
+	log.Print("6")
 
 	// Starting date
 	date := time.Now()
diff --git a/utils/methods.go b/utils/methods.go
index 2b54eaa..c4b1917 100644
--- a/utils/methods.go
+++ b/utils/methods.go
@@ -104,6 +104,7 @@ func RefreshToken(chromedpCtx context.Context) map[string][]string {
 
 // This function signs into Astra
 func RefreshAstraToken(chromedpCtx context.Context) map[string][]string {
+	log.Print("5.1")
 	// Get username and password
 	username, present := os.LookupEnv("LOGIN_ASTRA_USERNAME")
 	if !present {
@@ -113,6 +114,7 @@ func RefreshAstraToken(chromedpCtx context.Context) map[string][]string {
 	if !present {
 		log.Panic("LOGIN_ASTRA_PASSWORD is missing from .env!")
 	}
+	log.Print("5.2")
 
 	// Sign in
 	VPrintf("Signing in...")
@@ -132,6 +134,7 @@ func RefreshAstraToken(chromedpCtx context.Context) map[string][]string {
 	if err != nil {
 		panic(err)
 	}
+	log.Print("5.3")
 
 	// Save all cookies to string
 	cookieStr := ""
@@ -156,6 +159,7 @@ func RefreshAstraToken(chromedpCtx context.Context) map[string][]string {
 	if err != nil {
 		panic(err)
 	}
+	log.Print("5.4")
 
 	// Return headers, copied from a request the actual site made
 	return map[string][]string{

From 74315301c4b9e038b993f51194f592264b07fa51 Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 23:40:47 -0600
Subject: [PATCH 18/20] More print

---
 utils/methods.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/utils/methods.go b/utils/methods.go
index c4b1917..84e5f0a 100644
--- a/utils/methods.go
+++ b/utils/methods.go
@@ -144,18 +144,22 @@ func RefreshAstraToken(chromedpCtx context.Context) map[string][]string {
 			cookies, err := network.GetCookies().Do(ctx)
 			gotToken := false
 			for _, cookie := range cookies {
+				log.Printf("5.3.1 %s=%s; ", cookie.Name, cookie.Value)
 				cookieStr = fmt.Sprintf("%s%s=%s; ", cookieStr, cookie.Name, cookie.Value)
 				if cookie.Name == "UTXDallas.ASPXFORMSAUTH" {
-					VPrintf("Got new token: PTGSESSID = %s", cookie.Value)
+					VPrintf("Got new token: UTXDallas.ASPXFORMSAUTH = %s", cookie.Value)
 					gotToken = true
 				}
+				log.Print("5.3.2")
 			}
+			log.Print(gotToken)
 			if !gotToken {
 				return errors.New("failed to get a new token")
 			}
 			return err
 		}),
 	)
+	log.Print("5.3.3")
 	if err != nil {
 		panic(err)
 	}

From d30caf4a4f7b886132aaadd0da76407aa1703bae Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 23:47:16 -0600
Subject: [PATCH 19/20] Change error handling

---
 utils/methods.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/utils/methods.go b/utils/methods.go
index 84e5f0a..806bc4d 100644
--- a/utils/methods.go
+++ b/utils/methods.go
@@ -142,6 +142,9 @@ func RefreshAstraToken(chromedpCtx context.Context) map[string][]string {
 		chromedp.WaitVisible(`body`, chromedp.ByQuery),
 		chromedp.ActionFunc(func(ctx context.Context) error {
 			cookies, err := network.GetCookies().Do(ctx)
+			if err != nil {
+				return err
+			}
 			gotToken := false
 			for _, cookie := range cookies {
 				log.Printf("5.3.1 %s=%s; ", cookie.Name, cookie.Value)
@@ -156,7 +159,7 @@ func RefreshAstraToken(chromedpCtx context.Context) map[string][]string {
 			if !gotToken {
 				return errors.New("failed to get a new token")
 			}
-			return err
+			return nil
 		}),
 	)
 	log.Print("5.3.3")

From ac835b6f0ed8a05a2923dfcd801a4faeeeffbdb8 Mon Sep 17 00:00:00 2001
From: Tyler Hill <gotyhill@gmail.com>
Date: Tue, 18 Feb 2025 23:53:15 -0600
Subject: [PATCH 20/20] Yet more print

---
 utils/methods.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/utils/methods.go b/utils/methods.go
index 806bc4d..87ce9de 100644
--- a/utils/methods.go
+++ b/utils/methods.go
@@ -159,10 +159,12 @@ func RefreshAstraToken(chromedpCtx context.Context) map[string][]string {
 			if !gotToken {
 				return errors.New("failed to get a new token")
 			}
+			log.Print("5.3.3")
 			return nil
 		}),
+		chromedp.WaitVisible(`body`, chromedp.ByQuery),
 	)
-	log.Print("5.3.3")
+	log.Print("5.3.4")
 	if err != nil {
 		panic(err)
 	}