From b2625756c6e434ad4ebe56e0b14a4b1b778e021f Mon Sep 17 00:00:00 2001 From: elijahab Date: Tue, 24 Feb 2026 11:00:32 -0800 Subject: [PATCH 1/2] Issues 12 and 13, changed compression to zip. removed all but zip file from reports dur, sending only .tzt file over email --- gecko | 5 ++- triage_package/triage.ini | 47 +++++--------------- triage_package/triage_tool.py | 84 +++++++++++++++++++++-------------- 3 files changed, 65 insertions(+), 71 deletions(-) diff --git a/gecko b/gecko index a031271..0378b64 100755 --- a/gecko +++ b/gecko @@ -83,8 +83,9 @@ def run_triage(): gecko.grab_science_image() gecko.compress_report() - if gecko.email_alerts: - gecko.send_report() + if gecko.email_alerts: + gecko.send_report() + gecko.cleanup_reports_dir() print("\nTriage workflow complete!") else: diff --git a/triage_package/triage.ini b/triage_package/triage.ini index a321c66..11d490c 100644 --- a/triage_package/triage.ini +++ b/triage_package/triage.ini @@ -4,23 +4,19 @@ help_text = (EXAMPLES) os: ubuntu os_version: 24.04 - - -os = -os_version = -initialized = +os = CentOS Linux +os_version = 8 +initialized = true [Logs] help_text = - Please provide the path to your logs and science directory + Please provide the path to your logs and science directory The logs and science directory should be composed of dated dir in UTC format for the program to iterate through. (EXAMPLES) In this example latest is symliked to the most current date logs_dir: /data/latest/logs/ science_dir: /data/latest/ - - -logs_dir = -science_dir = +logs_dir = /data/latest/logs/ +science_dir = /data/latest/ [Report] help_text = @@ -30,16 +26,11 @@ help_text = time_pattern: ^(\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?) email_alerts: false recipient_email: eng@observatory.edu -# sender_email: hello@gmail.com -# sender_password: password - - -report_path = +report_path = /home/observer/gecko/reports time_pattern = ^(\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?) -email_alerts = -recipient_email = -#sender_email = -#sender_password = +email_alerts = true +recipient_email = elijahab@caltech.edu +machine_name = NGPS-Machine [Machine] help_text = @@ -47,20 +38,6 @@ help_text = (EXAMPLES) cpu_threshold: 85 memory_threshold: 90 - - -cpu_threshold = -memory_threshold = +cpu_threshold = 80 +memory_threshold = 80 -#[VNC] -#help_text = -# Please provide VNC details: -# (EXAMPLES) -# host: host.provider.com/host.iden.edu -# password: Password1234 -# vnc_sessions: 1,2,3,4,5,12 - - -#host = -#password = -#vnc_sessions = diff --git a/triage_package/triage_tool.py b/triage_package/triage_tool.py index c27a4da..0e9dd5a 100755 --- a/triage_package/triage_tool.py +++ b/triage_package/triage_tool.py @@ -19,7 +19,7 @@ import os import sys import subprocess -import tarfile +import zipfile import shutil import configparser import smtplib @@ -45,7 +45,7 @@ def __init__(self, config: str, message:str = "", init = False): self.cutoff = datetime.now().replace(tzinfo=None) - timedelta(hours=24) self.message = message self.time_pattern = r"^(\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?)" - self.tar_filename = "" + self.zip_filename = "" # Create a ConfigParser object self.config = configparser.ConfigParser() @@ -109,7 +109,7 @@ def load_config(self, config): self.email_alerts = self.config.getboolean("Report", "email_alerts") if self.email_alerts: self.target_email = self.config["Report"]["recipient_email"] - #self.sender_email = self.config["Report"]["sender_email"] + self.machine_name = self.config["Report"]["machine_name"] #self.sender_password = self.config["Report"]["sender_password"] self.r_path = self.config["Report"]["report_path"] self.log_dir = self.config["Logs"]["logs_dir"] @@ -239,7 +239,7 @@ def find_displays(self): return sorted(displays, key=lambda d: int(d[1:])) def gather_logs(self): - ''' Gathers Logs from all paths to dump into tar comp. ''' + ''' Gathers Logs from all paths to dump into zip comp. ''' #Iterate through log paths in config file for subdir, dirs, files in os.walk(self.log_dir): # pylint: disable = W0612 for file in files: @@ -293,26 +293,26 @@ def comb_logs(self): print(f"An unexpected error occurred: {e}") def compress_report(self): - '''Compresses report file into a tar.gz format to be emailed''' - #Tar file and add it to message - #with tarfile.open(f"{self.reports_path}/gecko_{self.utc_date}.tar.gz", "w:gz") as tar: - # tar.add(f"{self.reports_path}", arcname=os.path.basename(f"{self.reports_path}")) + """Compresses report files into a .zip format to be emailed""" filename = f"{self.reports_path}/gecko_{self.utc_date}" - filename = filename.replace(" ", "_").replace('+','') - self.tar_filename = filename.replace(':', '').replace('.', '_') + ".tar.gz" - with tarfile.open(self.tar_filename, "w:gz") as tar: + filename = filename.replace(" ", "_").replace("+", "") + self.zip_filename = filename.replace(":", "").replace(".", "_") + ".zip" + + with zipfile.ZipFile(self.zip_filename, "w", compression=zipfile.ZIP_DEFLATED) as zipf: for root, dirs, files in os.walk(self.reports_path): for file in files: - # Only include files containing utc_time - if self.utc_time in file or ".log" in file: + # Only include files containing utc_time OR log files + if self.utc_time in file or file.endswith(".log"): full_path = os.path.join(root, file) - # Keep relative structure inside tar + # Keep relative structure inside zip arcname = os.path.relpath(full_path, self.reports_path) - tar.add(full_path, arcname=arcname) + zipf.write(full_path, arcname=arcname) + + print(f"Created ZIP file: {self.zip_filename}") def grab_science_image(self): '''Grabs most recent science image(s) to include in triage''' @@ -354,11 +354,19 @@ def send_report(self): # Create email msg = EmailMessage() msg['Subject'] = f'Gecko Report {self.utc_date}' - msg['From'] = self.sender_email # replace with actual sender + msg['From'] = self.machine_name # replace with actual sender msg['To'] = self.target_email # can be comma-separated string or list + zip_path = full_path = os.path.join(self.reports_path, self.zip_filename) # Email body - msg.set_content("Please see attached report images.") + body = ( + f"{self.message}\n\n" + "To view logs and images:\n" + " 1. Download the attached .zip file\n" + " 2. Extract it\n\n" + f"Compressed report location:\n{zip_path}" + ) + msg.set_content(body) #.txt file first with open(self.report_name, 'rb') as f: @@ -369,17 +377,17 @@ def send_report(self): filename=os.path.basename(self.report_name) ) - #tar.gz file next - with open(self.tar_filename, 'rb') as f: - msg.add_attachment( - f.read(), - maintype='application', - subtype='gzip', - filename=os.path.basename(self.tar_filename) - ) + #.zip file next + #with open(self.zip_filename, 'rb') as f: + # msg.add_attachment( + # f.read(), + # maintype='application', + # subtype='gzip', + # filename=os.path.basename(self.zip_filename) + # ) # Attach PNG images recursively from the reports_path - image_files = glob.glob(os.path.join(self.reports_path, '**', f'*{self.utc_time}.png'), recursive=True) + #image_files = glob.glob(os.path.join(self.reports_path, '**', f'*{self.utc_time}.png'), recursive=True) #for file in image_files: # with open(file, 'rb') as fp: # img_data = fp.read() @@ -396,14 +404,14 @@ def send_report(self): # smtp.send_message(msg) #print(f"Report sent to {self.target_email}") - for file in image_files: - with open(file, "rb") as fp: - msg.add_attachment( - fp.read(), - maintype="image", - subtype="png", - filename=os.path.basename(file) - ) + #for file in image_files: + # with open(file, "rb") as fp: + # msg.add_attachment( + # fp.read(), + # maintype="image", + # subtype="png", + # filename=os.path.basename(file) + # ) # Send using local sendmail instead of SMTP try: @@ -416,3 +424,11 @@ def send_report(self): except Exception as e: #pylint: disable=W0718 print("Failed to send report:", e) + + def cleanup_reports_dir(self): + """Clean up and rid of all files that are not a .zip file""" + for root, dirs, files in os.walk(self.reports_path): + for filename in files: + full_path = os.path.join(self.reports_path, filename) + if not full_path.endswith(".zip"): + os.remove(full_path) From 8019aea4a220c30fcf286c705d9541a511a75d49 Mon Sep 17 00:00:00 2001 From: elijahab Date: Tue, 24 Feb 2026 11:19:07 -0800 Subject: [PATCH 2/2] Saving the most recent version of NGPS gecko deployment without sensitive information --- triage_package/triage.ini | 48 ++++++++++++++++++++++++++--------- triage_package/triage_tool.py | 13 +++++----- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/triage_package/triage.ini b/triage_package/triage.ini index 11d490c..1ac7a87 100644 --- a/triage_package/triage.ini +++ b/triage_package/triage.ini @@ -4,19 +4,23 @@ help_text = (EXAMPLES) os: ubuntu os_version: 24.04 -os = CentOS Linux -os_version = 8 -initialized = true + + +os = +os_version = +initialized = [Logs] help_text = - Please provide the path to your logs and science directory + Please provide the path to your logs and science directory The logs and science directory should be composed of dated dir in UTC format for the program to iterate through. (EXAMPLES) In this example latest is symliked to the most current date logs_dir: /data/latest/logs/ science_dir: /data/latest/ -logs_dir = /data/latest/logs/ -science_dir = /data/latest/ + + +logs_dir = +science_dir = [Report] help_text = @@ -26,11 +30,17 @@ help_text = time_pattern: ^(\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?) email_alerts: false recipient_email: eng@observatory.edu -report_path = /home/observer/gecko/reports +# sender_email: hello@gmail.com +# sender_password: password + + +report_path = time_pattern = ^(\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?) -email_alerts = true -recipient_email = elijahab@caltech.edu -machine_name = NGPS-Machine +email_alerts = +recipient_email = +machine_name = +#sender_email = +#sender_password = [Machine] help_text = @@ -38,6 +48,20 @@ help_text = (EXAMPLES) cpu_threshold: 85 memory_threshold: 90 -cpu_threshold = 80 -memory_threshold = 80 + + +cpu_threshold = +memory_threshold = +#[VNC] +#help_text = +# Please provide VNC details: +# (EXAMPLES) +# host: host.provider.com/host.iden.edu +# password: Password1234 +# vnc_sessions: 1,2,3,4,5,12 + + +#host = +#password = +#vnc_sessions = diff --git a/triage_package/triage_tool.py b/triage_package/triage_tool.py index 0e9dd5a..cbcee2c 100755 --- a/triage_package/triage_tool.py +++ b/triage_package/triage_tool.py @@ -22,16 +22,15 @@ import zipfile import shutil import configparser -import smtplib -from pathlib import Path +from pathlib import Path #pylint: disable=W0611 from email.message import EmailMessage import re from datetime import datetime, timezone, timedelta import glob import threading #pylint: disable=W0611 import socket #pylint: disable=W0611 -import psutil -from vncdotool import api #pylint: disable=W0611 +import psutil #pylint: disable=E0401 +from vncdotool import api #pylint: disable=E0401,W0611 class Triagetools(object): """Triage tool for bug catching and error reporting""" @@ -300,7 +299,7 @@ def compress_report(self): self.zip_filename = filename.replace(":", "").replace(".", "_") + ".zip" with zipfile.ZipFile(self.zip_filename, "w", compression=zipfile.ZIP_DEFLATED) as zipf: - for root, dirs, files in os.walk(self.reports_path): + for root, dirs, files in os.walk(self.reports_path): # pylint: disable = W0612 for file in files: # Only include files containing utc_time OR log files @@ -356,7 +355,7 @@ def send_report(self): msg['Subject'] = f'Gecko Report {self.utc_date}' msg['From'] = self.machine_name # replace with actual sender msg['To'] = self.target_email # can be comma-separated string or list - zip_path = full_path = os.path.join(self.reports_path, self.zip_filename) + zip_path = os.path.join(self.reports_path, self.zip_filename) # Email body body = ( @@ -427,7 +426,7 @@ def send_report(self): def cleanup_reports_dir(self): """Clean up and rid of all files that are not a .zip file""" - for root, dirs, files in os.walk(self.reports_path): + for root, dirs, files in os.walk(self.reports_path): # pylint: disable = W0612 for filename in files: full_path = os.path.join(self.reports_path, filename) if not full_path.endswith(".zip"):