diff --git a/docs/nightly_reports.md b/docs/nightly_reports.md new file mode 100644 index 000000000..3a3e51e40 --- /dev/null +++ b/docs/nightly_reports.md @@ -0,0 +1,107 @@ +# Nightly Observation Reports + +## Overview + +POCS now automatically generates nightly observation reports that provide a comprehensive summary of each observing night. These reports are generated at the end of the night during the housekeeping state. + +## Report Contents + +Each nightly report includes: + +1. **Observations Summary** + - Total number of observation sequences + - Number of unique fields observed + - Details for each field including: + - Field name + - Start time + - Number of exposures taken vs. planned + +2. **Safety Summary** + - Status of all safety checks: + - AC Power Connected + - Dark Sky + - Weather Safe + - Root Disk Space + - Image Disk Space + - Reasons for non-observation (if any): + - Unsafe weather conditions + - Not dark enough for observations + - AC power issues + - Insufficient disk space + +## Configuration + +Reports are saved to a configurable directory. By default, they are saved to `~/reports`, but this can be customized in your POCS configuration: + +```yaml +directories: + reports: /path/to/your/reports/directory +``` + +## Report Files + +Reports are saved with the naming convention: +``` +nightly_report_YYYYMMDD.txt +``` + +For example: `nightly_report_20260213.txt` + +## Sample Report + +``` +================================================================================ +PANOPTES Nightly Observation Report - 2026-02-13 +================================================================================ + +OBSERVATIONS SUMMARY +-------------------------------------------------------------------------------- +Total observation sequences: 2 +Unique fields observed: 2 + +Observations by field: + - M42OrionNebula: 1 sequence(s) + * Started: 2026-02-13T01:30:00, Exposures: 0/30 + - M31Andromeda: 1 sequence(s) + * Started: 2026-02-13T03:15:00, Exposures: 0/40 + +SAFETY SUMMARY +-------------------------------------------------------------------------------- +Most recent safety check: + AC Power Connected...................... ✓ PASS + Dark Sky................................ ✓ PASS + Weather Safe............................ ✗ FAIL + Root Disk Space......................... ✓ PASS + Image Disk Space........................ ✓ PASS + +WARNING: Some safety checks failed. +Reasons for non-observation: + - Unsafe weather conditions + +================================================================================ +End of Report +================================================================================ +``` + +## Implementation Details + +The nightly report feature is implemented in: +- `src/panoptes/pocs/utils/report.py` - Core report generation logic +- `src/panoptes/pocs/state/states/default/housekeeping.py` - Integration with state machine + +The report generator queries the POCS database for: +- Observed list from the scheduler +- Most recent safety check data + +Reports are generated automatically during the housekeeping state at the end of each night, before the observed list is reset. + +## Testing + +Comprehensive tests are included in: +- `tests/utils/test_report.py` - Unit tests for report generation +- `tests/test_housekeeping_state.py` - Integration tests + +Run tests with: +```bash +pytest tests/utils/test_report.py tests/test_housekeeping_state.py -v +``` diff --git a/src/panoptes/pocs/state/states/default/housekeeping.py b/src/panoptes/pocs/state/states/default/housekeeping.py index e93c3ddc1..89b3eed58 100644 --- a/src/panoptes/pocs/state/states/default/housekeeping.py +++ b/src/panoptes/pocs/state/states/default/housekeeping.py @@ -1,3 +1,9 @@ +from datetime import datetime +from pathlib import Path + +from panoptes.pocs.utils.report import NightlyReport + + def on_enter(event_data): """ """ pocs = event_data.model @@ -5,6 +11,34 @@ def on_enter(event_data): pocs.say("Resetting the list of observations and doing some cleanup!") + # Generate nightly report before cleanup + try: + observed_list = pocs.observatory.scheduler.observed_list + + # Create report + report_generator = NightlyReport(db=pocs.db) + + # Get reports directory from config or use default + reports_dir = Path( + pocs.get_config("directories.reports", default="~/reports") + ).expanduser() + + # Generate filename with date + date_str = datetime.now().strftime("%Y%m%d") + report_path = reports_dir / f"nightly_report_{date_str}.txt" + + # Generate and save report + report_text = report_generator.generate_report( + observed_list=observed_list, + output_path=report_path + ) + + pocs.logger.info(f"Nightly report generated: {report_path}") + pocs.say(f"Generated nightly report with {len(observed_list)} observation(s)") + + except Exception as e: # pragma: no cover + pocs.logger.warning(f"Problem generating nightly report: {e!r}") + # Cleanup existing observations try: pocs.observatory.scheduler.reset_observed_list() diff --git a/src/panoptes/pocs/utils/report.py b/src/panoptes/pocs/utils/report.py new file mode 100644 index 000000000..4fce3dece --- /dev/null +++ b/src/panoptes/pocs/utils/report.py @@ -0,0 +1,188 @@ +"""Nightly report generation for POCS observations. + +This module provides utilities to generate end-of-night reports summarizing +observations taken and reasons for non-observation (weather, errors, etc.). +""" +from collections import defaultdict +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Optional + +from panoptes.pocs.base import PanBase + + +class NightlyReport(PanBase): + """Generate nightly observation reports. + + Creates a summary report at the end of each observing night that includes: + - List of observations successfully taken + - Count of exposures per field + - Reasons for non-observation (weather, safety, errors) + - Safety check statistics + """ + + def __init__(self, db=None, *args, **kwargs): + """Initialize the nightly report generator. + + Args: + db: Database instance to query for observation and safety data. + """ + super().__init__(*args, **kwargs) + self.db = db + + def generate_report( + self, + observed_list: Optional[Dict] = None, + output_path: Optional[Path] = None, + date: Optional[str] = None + ) -> str: + """Generate a comprehensive nightly report. + + Args: + observed_list: Dictionary of observations from the scheduler's observed_list. + output_path: Optional path to save the report to a file. + date: Optional date string for the report. Defaults to current date. + + Returns: + str: The formatted report text. + """ + if date is None: + date = datetime.now().strftime("%Y-%m-%d") + + report_lines = [] + report_lines.append("=" * 80) + report_lines.append(f"PANOPTES Nightly Observation Report - {date}") + report_lines.append("=" * 80) + report_lines.append("") + + # Observations summary + obs_summary = self._summarize_observations(observed_list) + report_lines.extend(obs_summary) + report_lines.append("") + + # Safety summary + safety_summary = self._summarize_safety_checks() + report_lines.extend(safety_summary) + report_lines.append("") + + report_lines.append("=" * 80) + report_lines.append("End of Report") + report_lines.append("=" * 80) + + report_text = "\n".join(report_lines) + + # Save to file if path provided + if output_path: + output_path = Path(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(report_text) + self.logger.info(f"Report saved to {output_path}") + + return report_text + + def _summarize_observations(self, observed_list: Optional[Dict] = None) -> List[str]: + """Summarize observations taken during the night. + + Args: + observed_list: Dictionary of observations from scheduler. + + Returns: + List[str]: Lines of text summarizing observations. + """ + lines = ["OBSERVATIONS SUMMARY", "-" * 80] + + if not observed_list or len(observed_list) == 0: + lines.append("No observations were completed during this night.") + return lines + + # Group observations by field + field_stats = defaultdict(lambda: {"count": 0, "seq_times": []}) + + for seq_time, observation in observed_list.items(): + field_name = observation.field.field_name + field_stats[field_name]["count"] += 1 + field_stats[field_name]["seq_times"].append(seq_time) + + lines.append(f"Total observation sequences: {len(observed_list)}") + lines.append(f"Unique fields observed: {len(field_stats)}") + lines.append("") + lines.append("Observations by field:") + + for field_name, stats in sorted(field_stats.items()): + lines.append(f" - {field_name}: {stats['count']} sequence(s)") + for seq_time in stats["seq_times"]: + obs = observed_list[seq_time] + exp_count = getattr(obs, 'current_exp_num', 0) + min_exp = getattr(obs, 'min_nexp', 0) + lines.append(f" * Started: {seq_time}, Exposures: {exp_count}/{min_exp}") + + return lines + + def _summarize_safety_checks(self) -> List[str]: + """Summarize safety check results during the night. + + Returns: + List[str]: Lines of text summarizing safety checks. + """ + lines = ["SAFETY SUMMARY", "-" * 80] + + if self.db is None: + lines.append("Database not available for safety summary.") + return lines + + try: + # Get the most recent safety record + safety_record = self.db.get_current("safety") + + if safety_record is None: + lines.append("No safety data available.") + return lines + + # Extract the actual data from the record + safety_data = safety_record.get("data", safety_record) + + lines.append("Most recent safety check:") + + # List each safety check and its status + safety_checks = { + "ac_power": "AC Power Connected", + "is_dark": "Dark Sky", + "good_weather": "Weather Safe", + "free_space_root": "Root Disk Space", + "free_space_images": "Image Disk Space" + } + + all_safe = True + for key, label in safety_checks.items(): + if key in safety_data: + status = "✓ PASS" if safety_data[key] else "✗ FAIL" + lines.append(f" {label:.<40} {status}") + if not safety_data[key]: + all_safe = False + + lines.append("") + if all_safe: + lines.append("All safety checks passed.") + else: + lines.append("WARNING: Some safety checks failed.") + lines.append("Reasons for non-observation:") + + failure_reasons = [] + if not safety_data.get("good_weather", True): + failure_reasons.append(" - Unsafe weather conditions") + if not safety_data.get("is_dark", True): + failure_reasons.append(" - Not dark enough for observations") + if not safety_data.get("ac_power", True): + failure_reasons.append(" - AC power disconnected") + if not safety_data.get("free_space_root", True): + failure_reasons.append(" - Insufficient disk space (root)") + if not safety_data.get("free_space_images", True): + failure_reasons.append(" - Insufficient disk space (images)") + + lines.extend(failure_reasons) + + except Exception as e: + lines.append(f"Error retrieving safety data: {e!r}") + self.logger.warning(f"Error in safety summary: {e!r}") + + return lines diff --git a/tests/test_housekeeping_state.py b/tests/test_housekeeping_state.py new file mode 100644 index 000000000..2ff0232e7 --- /dev/null +++ b/tests/test_housekeeping_state.py @@ -0,0 +1,149 @@ +"""Tests for housekeeping state with nightly report generation.""" +import pytest +from pathlib import Path +from collections import OrderedDict + +from panoptes.pocs.state.states.default import housekeeping +from panoptes.pocs.scheduler.observation.base import Observation +from panoptes.pocs.scheduler.field import Field +from panoptes.utils.database import PanDB + + +@pytest.fixture +def db(): + """Create a test database instance.""" + return PanDB(db_type="memory", db_name="test_housekeeping") + + +class MockEventData: + """Mock event data for state testing.""" + def __init__(self, pocs): + self.model = pocs + + +class MockScheduler: + """Mock scheduler for testing.""" + def __init__(self): + self.observed_list = OrderedDict() + + def reset_observed_list(self): + """Reset the observed list.""" + self.observed_list = OrderedDict() + + +class MockObservatory: + """Mock observatory for testing.""" + def __init__(self): + self.scheduler = MockScheduler() + + +class MockLogger: + """Mock logger for testing.""" + def info(self, message): + pass + + def warning(self, message): + pass + + def debug(self, message): + pass + + +class MockPOCS: + """Mock POCS instance for testing.""" + def __init__(self, db, config_dir): + self.db = db + self.observatory = MockObservatory() + self.next_state = None + self._config_dir = config_dir + self.messages = [] + self.logger = MockLogger() + + def say(self, message): + """Record messages.""" + self.messages.append(message) + + def get_config(self, key, default=None): + """Get config value.""" + if key == "directories.reports": + return self._config_dir / "reports" + return default + + +def test_housekeeping_generates_report(db, tmp_path): + """Test that housekeeping state generates a nightly report.""" + # Create a mock POCS instance + pocs = MockPOCS(db=db, config_dir=tmp_path) + + # Add some observations to the scheduler + field = Field(name="Test Field", position="20h00m00s +30d00m00s") + obs = Observation(field=field, exptime=120, min_nexp=10) + obs.seq_time = "2026-02-13T01:00:00" + pocs.observatory.scheduler.observed_list[obs.seq_time] = obs + + # Create mock event data + event_data = MockEventData(pocs) + + # Call housekeeping on_enter + housekeeping.on_enter(event_data) + + # Check that next_state was set + assert pocs.next_state == "sleeping" + + # Check that observed_list was reset + assert len(pocs.observatory.scheduler.observed_list) == 0 + + # Check that report file was created + reports_dir = tmp_path / "reports" + assert reports_dir.exists() + + # Check for report files (there should be at least one) + report_files = list(reports_dir.glob("nightly_report_*.txt")) + assert len(report_files) >= 1 + + # Read the report and verify contents + report_text = report_files[0].read_text() + assert "PANOPTES Nightly Observation Report" in report_text + assert "OBSERVATIONS SUMMARY" in report_text + assert "SAFETY SUMMARY" in report_text + + +def test_housekeeping_handles_empty_observations(db, tmp_path): + """Test housekeeping with no observations.""" + # Create a mock POCS instance with no observations + pocs = MockPOCS(db=db, config_dir=tmp_path) + + # Create mock event data + event_data = MockEventData(pocs) + + # Call housekeeping on_enter + housekeeping.on_enter(event_data) + + # Check that next_state was set + assert pocs.next_state == "sleeping" + + # Check that report was still generated + reports_dir = tmp_path / "reports" + assert reports_dir.exists() + + report_files = list(reports_dir.glob("nightly_report_*.txt")) + assert len(report_files) >= 1 + + # Verify the report mentions no observations + report_text = report_files[0].read_text() + assert "No observations were completed" in report_text + + +def test_housekeeping_handles_errors_gracefully(tmp_path): + """Test that housekeeping continues even if report generation fails.""" + # Create a mock POCS with invalid db (None) + pocs = MockPOCS(db=None, config_dir=tmp_path) + + # Create mock event data + event_data = MockEventData(pocs) + + # This should not raise an exception + housekeeping.on_enter(event_data) + + # Check that next_state was still set despite error + assert pocs.next_state == "sleeping" diff --git a/tests/utils/test_report.py b/tests/utils/test_report.py new file mode 100644 index 000000000..f17045803 --- /dev/null +++ b/tests/utils/test_report.py @@ -0,0 +1,194 @@ +"""Tests for nightly report generation.""" +import pytest +from pathlib import Path +from datetime import datetime +from collections import OrderedDict + +from panoptes.pocs.utils.report import NightlyReport +from panoptes.pocs.scheduler.observation.base import Observation +from panoptes.pocs.scheduler.field import Field +from panoptes.utils.database import PanDB + + +@pytest.fixture +def db(): + """Create a test database instance.""" + return PanDB(db_type="memory", db_name="test_report") + + +@pytest.fixture +def report_generator(db): + """Create a NightlyReport instance with the test database.""" + return NightlyReport(db=db) + + +@pytest.fixture +def sample_observations(): + """Create sample observations for testing.""" + observed_list = OrderedDict() + + # Create a simple field and observation + field1 = Field(name="Test Field 1", position="20h00m00s +30d00m00s") + obs1 = Observation(field=field1, exptime=120, min_nexp=10) + obs1.seq_time = "2026-02-13T01:00:00" + + field2 = Field(name="Test Field 2", position="22h00m00s +40d00m00s") + obs2 = Observation(field=field2, exptime=120, min_nexp=20) + obs2.seq_time = "2026-02-13T02:30:00" + + observed_list[obs1.seq_time] = obs1 + observed_list[obs2.seq_time] = obs2 + + return observed_list + + +def test_report_generator_creation(report_generator): + """Test that NightlyReport can be created.""" + assert report_generator is not None + assert report_generator.db is not None + + +def test_generate_report_empty(report_generator, tmp_path): + """Test report generation with no observations.""" + output_path = tmp_path / "test_report.txt" + + report_text = report_generator.generate_report( + observed_list={}, + output_path=output_path + ) + + assert "PANOPTES Nightly Observation Report" in report_text + assert "No observations were completed" in report_text + assert output_path.exists() + assert output_path.read_text() == report_text + + +def test_generate_report_with_observations(report_generator, sample_observations, tmp_path): + """Test report generation with observations.""" + output_path = tmp_path / "test_report.txt" + + report_text = report_generator.generate_report( + observed_list=sample_observations, + output_path=output_path + ) + + assert "PANOPTES Nightly Observation Report" in report_text + # Field names have spaces removed, so "Test Field 1" becomes "TestField1" + assert "TestField1" in report_text + assert "TestField2" in report_text + assert "Total observation sequences: 2" in report_text + assert "Unique fields observed: 2" in report_text + assert output_path.exists() + + +def test_generate_report_without_file(report_generator, sample_observations): + """Test report generation without saving to file.""" + report_text = report_generator.generate_report( + observed_list=sample_observations + ) + + assert "PANOPTES Nightly Observation Report" in report_text + # Field names have spaces removed + assert "TestField1" in report_text + + +def test_summarize_observations_empty(report_generator): + """Test observation summary with no observations.""" + summary = report_generator._summarize_observations({}) + + summary_text = "\n".join(summary) + assert "OBSERVATIONS SUMMARY" in summary_text + assert "No observations were completed" in summary_text + + +def test_summarize_observations_with_data(report_generator, sample_observations): + """Test observation summary with observation data.""" + summary = report_generator._summarize_observations(sample_observations) + + summary_text = "\n".join(summary) + assert "OBSERVATIONS SUMMARY" in summary_text + # Field names have spaces removed + assert "TestField1" in summary_text + assert "TestField2" in summary_text + assert "2 sequence(s)" in summary_text or "1 sequence(s)" in summary_text + + +def test_summarize_safety_no_data(report_generator): + """Test safety summary with no safety data in database.""" + summary = report_generator._summarize_safety_checks() + + summary_text = "\n".join(summary) + assert "SAFETY SUMMARY" in summary_text + assert "No safety data available" in summary_text + + +def test_summarize_safety_all_safe(report_generator, db): + """Test safety summary with all checks passing.""" + # Insert safe conditions into database + safety_data = { + "ac_power": True, + "is_dark": True, + "good_weather": True, + "free_space_root": True, + "free_space_images": True + } + db.insert_current("safety", safety_data) + + summary = report_generator._summarize_safety_checks() + + summary_text = "\n".join(summary) + assert "SAFETY SUMMARY" in summary_text + assert "All safety checks passed" in summary_text + assert "✓ PASS" in summary_text + + +def test_summarize_safety_with_failures(report_generator, db): + """Test safety summary with failing checks.""" + # Insert unsafe conditions into database + safety_data = { + "ac_power": True, + "is_dark": False, + "good_weather": False, + "free_space_root": True, + "free_space_images": True + } + db.insert_current("safety", safety_data) + + summary = report_generator._summarize_safety_checks() + + summary_text = "\n".join(summary) + assert "SAFETY SUMMARY" in summary_text + assert "WARNING: Some safety checks failed" in summary_text + assert "✗ FAIL" in summary_text + assert "Unsafe weather conditions" in summary_text + assert "Not dark enough for observations" in summary_text + + +def test_report_date_format(report_generator): + """Test that report includes proper date formatting.""" + report_text = report_generator.generate_report(observed_list={}) + + # Should have a date in YYYY-MM-DD format + today = datetime.now().strftime("%Y-%m-%d") + assert today in report_text + + +def test_multiple_observations_same_field(report_generator): + """Test handling of multiple observations of the same field.""" + observed_list = OrderedDict() + + field = Field(name="Repeated Field", position="20h00m00s +30d00m00s") + + for i in range(3): + obs = Observation(field=field, exptime=120, min_nexp=10) + seq_time = f"2026-02-13T0{i}:00:00" + obs.seq_time = seq_time + observed_list[seq_time] = obs + + summary = report_generator._summarize_observations(observed_list) + summary_text = "\n".join(summary) + + # Field name has spaces removed: "Repeated Field" becomes "RepeatedField" + assert "RepeatedField: 3 sequence(s)" in summary_text + assert "Total observation sequences: 3" in summary_text + assert "Unique fields observed: 1" in summary_text