diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 740f7bc..398fdac 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -8,24 +8,6 @@ on: name: Create GitHub Release jobs: - build: - name: Create Release - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} - body: | - See Changelog for details - draft: false - prerelease: false deploy: name: Push Release to PyPi runs-on: ubuntu-latest diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..2d04c35 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,109 @@ +name: Build, test, and publish documentation + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +# Required permissions for deploying to GitHub Pages +permissions: + contents: write + pages: write + +jobs: + test: + name: Run tests and upload coverage + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.12] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: "pip-${{ runner.os }}-py-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }}" + restore-keys: | + pip-${{ runner.os }}-py-${{ matrix.python-version }}- + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + pip install -r docs/requirements.txt + pip install -e . + pip install pytest pytest-cov + + - name: Run tests with coverage + run: | + pytest --cov=panoptes.data --cov-report=xml:coverage.xml --cov-report=html:htmlcov -q + + - name: Upload coverage report (xml) + uses: actions/upload-artifact@v4 + with: + name: coverage-xml + path: coverage.xml + + - name: Upload coverage HTML + uses: actions/upload-artifact@v4 + with: + name: coverage-html + path: htmlcov + + build-docs: + name: Build and publish docs + needs: test + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.12] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: "pip-${{ runner.os }}-py-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }}" + restore-keys: | + pip-${{ runner.os }}-py-${{ matrix.python-version }}- + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + pip install -r docs/requirements.txt + pip install -e . + + - name: Build docs + working-directory: docs + run: make html + + - name: Upload HTML docs artifact + uses: actions/upload-artifact@v4 + with: + name: docs-html + path: docs/_build/html + + - name: Deploy to GitHub Pages + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: docs/_build/html + publish_branch: gh-pages diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..03b9c7d --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,52 @@ +name: Run tests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + test: + name: Run test suite + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.12] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: "pip-${{ runner.os }}-py-${{ matrix.python-version }}-${{ hashFiles('**/requirements*.txt') }}" + restore-keys: | + pip-${{ runner.os }}-py-${{ matrix.python-version }}- + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + # Install broad dependencies used by the project/tests + if [ -f docs/requirements.txt ]; then pip install -r docs/requirements.txt; fi + # Install the package itself + pip install -e . + # Install test tools + pip install pytest pytest-cov + + - name: Run tests with coverage + run: | + pytest --cov=panoptes.data --cov-report=xml:coverage.xml --cov-report=term-missing -q + + - name: Upload coverage (xml) + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-xml + path: coverage.xml diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index 072151f..0000000 --- a/.isort.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[settings] -profile = black -known_first_party = panoptes diff --git a/.readthedocs.yaml b/.readthedocs.yaml index aed8e51..72e839d 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,21 +1,13 @@ -# .readthedocs.yaml -# Read the Docs configuration file -# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details - -# Required version: 2 -# Set the OS, Python version and other tools you might need build: os: ubuntu-22.04 tools: python: "3.12" -# Build documentation in the "docs/" directory with Sphinx sphinx: configuration: docs/conf.py -# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html python: install: - requirements: docs/requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 33cc6f0..1d3cc21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,41 @@ # Changelog +## UNRELEASED + +- Modernize the repo to use `pyproject.toml`. +- Update docs. +- Add basic tests. +- Releases are created directly by GitHub Actions. + +## Version 0.2.0 (2024-10-08) + +- Print length of observations. +- Only apply status filter if given. + +## Version 0.1.9 (2024-10-07) + +- Do better path matching for image url for new scheme. + +## Version 0.1.8 (2024-10-07) + +- Merge branch 'main' of github.com:panoptes/panoptes-data + +## Version 0.1.7 (2024-04-18) + +- Fix image_list for latest processing. + +## Version 0.1.6 (2024-04-18) + +- Change default bucket for downloading images and also make it a parameter. + +## Version 0.1.5 (2024-02-02) + +- Fixing release for pypi api tokens. + +## Version 0.1.4 (2024-02-02) + +- Add GitHub Actions PyPI release workflow. + ## Version 0.1.3 - Cleanup of images and observations. @@ -44,7 +80,15 @@ - Added basic cli interface for download images and metadata for observations. - Fixed install dependencies. +## Version 0.0.3 (2022-06-24) + +- More cleanup of dependencies for release. + ## Version 0.0.2 - Observation search available via ``panoptes.data.search.search_observations``. - ``ObservationInfo`` for working with observation data and metadata. + +## Version 0.0.1 (2022-06-23) + +- Fixing setup. diff --git a/README.md b/README.md index a4b3f06..463cd79 100644 --- a/README.md +++ b/README.md @@ -14,20 +14,29 @@ pip install panoptes-data ## Examples -> See example Jupyter Notebooks in the [`notebooks`](notebooks/) folder. +See the example Jupyter Notebooks in the `notebooks/` directory. ### Finding observations ```py ->>> from panoptes.data.search import search_observations ->>> from panoptes.data.observations import ObservationInfo +from panoptes.data.search import search_observations +from panoptes.data.observations import ObservationInfo ->>> # Find some observations ->>> results = search_observations(by_name='M42') +# Find some observations +results = search_observations(by_name='M42') ->>> # Use last result entry to create ObservationInfo object. ->>> obs_info = ObservationInfo(meta=results.iloc[0]) ->>> obs_info.meta +# Use last result entry to create ObservationInfo object. +obs_info = ObservationInfo(meta=results.iloc[0]) +print(obs_info.meta) + +# Create an ObservationInfo object directly from a sequence_id. +obs_info = ObservationInfo('PAN001_14d3bd_20180113T052325') +# But then there is no metadata: +print(obs_info.meta) +``` + +```text +Sample output (truncated): camera_id 14d3bd camera_lens_serial_number HA0028608 @@ -43,13 +52,6 @@ time 2018-01-13 05:23:25+00:00 total_exptime 3360.0 unit_id PAN001 Name: 6121, dtype: object - ->>> # Create an ObservationInfo object directly from a sequence_id. ->>> obs_info = ObservationInfo('PAN001_14d3bd_20180113T052325') ->>> # But then there is no metadata: ->>> obs_info.meta - -{} ``` ### Downloading images @@ -57,7 +59,7 @@ Name: 6121, dtype: object The `ObservationInfo` object makes it easy to download the files: ```py ->>> obs_info.download_images() +obs_info.download_images() ``` ### Command-line tools @@ -67,8 +69,12 @@ There is a simple command line tool that allows for both searching and downloadi #### Search for observations: ```bash -$ panoptes-data search --name M42 --min-num-images 90 +panoptes-data search --name M42 --min-num-images 90 +``` + +Example table output: +```text | sequence_id | field_name | unit_id | coordinates_mount_ra | coordinates_mount_dec | num_images | exptime | total_exptime | time | |:------------------------------|:-------------|:----------|-----------------------:|------------------------:|-------------:|----------:|----------------:|:--------------------------| | PAN022_977c86_20220108T090553 | M42 | PAN022 | 83.8221 | -5.39111 | 95 | 90 | 8550 | 2022-01-08 09:05:53+00:00 | diff --git a/docs/conf.py b/docs/conf.py index 3420a0c..57fe0ba 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,8 +8,8 @@ # serve to show the default. import os -import sys import shutil +import sys # -- Path setup -------------------------------------------------------------- @@ -52,7 +52,7 @@ apidoc.main(args) except Exception as e: - print("Running `sphinx-apidoc` failed!\n{}".format(e)) + print(f"Running `sphinx-apidoc` failed!\n{e}") # -- General configuration --------------------------------------------------- @@ -105,7 +105,7 @@ # General information about the project. project = "panoptes-data" -copyright = "2023, Wilfred Tyler Gee" +copyright = "2025, Wilfred Tyler Gee" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -165,6 +165,9 @@ # If this is True, todo emits a warning for each TODO entries. The default is False. todo_emit_warnings = True +# Mock heavy or optional imports so autodoc can generate docs without them. +autodoc_mock_imports = ["pydantic", "pydantic_settings", "tqdm", "typer"] + # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for diff --git a/docs/index.md b/docs/index.md index d8328f5..22aaef8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,6 +3,18 @@ :relative-images: ``` +## User documentation + +```{toctree} +:maxdepth: 2 + +readme +authors +contributing +changelog +license +``` + ## API reference ```{toctree} diff --git a/docs/readme.md b/docs/readme.md index 2cb706b..df9534c 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -2,3 +2,70 @@ :relative-docs: docs/ :relative-images: ``` + +# Building the documentation + +Follow these steps to build the HTML documentation locally. + +1. Create and activate a virtual environment (recommended): + +```bash +python3 -m venv .venv +. .venv/bin/activate +``` + +2. Upgrade packaging tools and install the docs requirements: + +```bash +python -m pip install --upgrade pip setuptools wheel +pip install -r docs/requirements.txt +``` + +3. For more complete API docs (fewer autodoc import warnings), install the project runtime dependencies in the same environment. This will allow Sphinx to import modules like `astropy` and `typer` during the build: + +```bash +# Install the package and its dependencies (editable install is convenient during development) +pip install -e . +# or install specific packages if you prefer: +# pip install astropy typer +``` + +4. Build the docs and open them locally: + +```bash +cd docs +make html +# open the index in your browser +open _build/html/index.html +``` + +Notes +- If you prefer not to install heavy runtime packages, `docs/conf.py` includes `autodoc_mock_imports` for some common heavy dependencies so the docs can still build; however installing the real packages gives more complete API documentation. +- If you see autodoc import warnings mentioning other missing packages, install those packages into your `.venv` or add them to `autodoc_mock_imports` in `docs/conf.py`. + +# CI / Automated docs build + +For CI (for example GitHub Actions) it's convenient to use a full docs requirements file that includes runtime packages needed for autodoc. A sample workflow is provided at `.github/workflows/docs.yml`. + +Quick CI steps (what the workflow runs): + +```bash +# create and activate a venv (CI runners already provide Python environment) +python3 -m venv .venv +. .venv/bin/activate + +# install full docs deps +python -m pip install --upgrade pip setuptools wheel +pip install -r docs/requirements.txt + +# install the package (editable) so autodoc can import it +pip install -e . + +# build +cd docs +make html +``` + +The CI example workflow will upload the generated `_build/html` directory as an artifact so you can preview the generated site. + +(See `.github/workflows/docs.yml` for the exact GitHub Actions configuration.) diff --git a/docs/requirements.txt b/docs/requirements.txt index 5d3210c..ea99621 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,3 +5,10 @@ linkify-it-py myst-parser pydata-sphinx-theme sphinx>=3.2.1 +pandas +astropy +pydantic +typer +tqdm +panoptes-utils[images] +photutils diff --git a/environment.yaml b/environment.yaml deleted file mode 100644 index 0e005b2..0000000 --- a/environment.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: panoptes-data -channels: - - https://conda.anaconda.org/conda-forge -dependencies: - - astropy - - numpy - - pandas - - pip - - pydantic - - pydantic-settings - - python-dateutil - - rich[jupyter] - - tabulate - - tqdm - - typer - - pip: - - . - - panoptes-utils[images] diff --git a/pyproject.toml b/pyproject.toml index f6935d6..8a45a39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,74 @@ [build-system] -# AVOID CHANGING REQUIRES: IT WILL BE UPDATED BY PYSCAFFOLD! -requires = ["setuptools>=46.1.0", "setuptools_scm[toml]>=5", "wheel"] -build-backend = "setuptools.build_meta" - -[tool.setuptools_scm] -# For smarter version schemes and other configuration options, -# check out https://github.com/pypa/setuptools_scm -version_scheme = "no-guess-dev" +requires = ["hatchling>=1.18", "hatch-vcs>=0.3"] +build-backend = "hatchling.build" + +[project] +name = "panoptes-data" +description = "Tools for working with PANOPTES data." +readme = "README.md" +requires-python = ">=3.12" +license = { file = "LICENSE.txt" } +authors = [ + { name = "Wilfred Tyler Gee", email = "wtylergee@gmail.com" }, +] +keywords = ["astronomy", "panoptes", "data"] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.12", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +dependencies = [ + "astropy", + "numpy", + "pandas", + "panoptes-utils[images]", + "pydantic", + "pydantic-settings", + "python-dateutil", + "rich[jupyter]", + "tabulate", + "tqdm", + "typer", +] +dynamic = ["version"] + +[project.urls] +Documentation = "https://projectpanoptes.org/" +Source = "https://github.com/panoptes/panoptes-data/" +Download = "https://pypi.org/project/panoptes-data/#files" + +[project.scripts] +panoptes-data = "panoptes.data.utils.cli.main:app" + +[tool.hatch.version] +source = "vcs" + +[tool.hatch.build] +include = ["src/panoptes/**", "README.md", "LICENSE.txt"] + +[tool.hatch.build.targets.wheel] +packages = ["src/panoptes"] + +[tool.hatch.envs.default] +dependencies = [ + "ruff>=0.5.0" +] +[tool.hatch.envs.default.scripts] +lint = "ruff check ." +lint-fix = "ruff check --fix ." +fmt = "ruff format ." +fmt-check = "ruff format --check ." + +[tool.ruff] +line-length = 100 +target-version = "py312" +src = ["src"] + +[tool.pytest.ini_options] +addopts = ["--cov=panoptes.data", "--cov-report", "term-missing", "--verbose"] +norecursedirs = ["dist", "build", ".tox"] +testpaths = ["tests"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 871b3c6..0000000 --- a/setup.cfg +++ /dev/null @@ -1,129 +0,0 @@ -# This file is used to configure your project. -# Read more about the various options under: -# https://setuptools.pypa.io/en/latest/userguide/declarative_config.html -# https://setuptools.pypa.io/en/latest/references/keywords.html - -[metadata] -name = panoptes-data -description = Tools for working with PANOPTES data. -author = Wilfred Tyler Gee -author_email = wtylergee@gmail.com -license = MIT -license_files = LICENSE.txt -long_description = file: README.md -long_description_content_type = text/markdown; charset=UTF-8 -url = https://github.com/panoptes/panoptes-data -# Add here related links, for example: -project_urls = - Documentation = https://projectpanoptes.org/ - Source = https://github.com/panoptes/panoptes-data/ - Download = https://pypi.org/project/panoptes-data/#files - -# Change if running only on Windows, Mac or Linux (comma-separated) -platforms = any - -# Add here all kinds of additional classifiers as defined under -# https://pypi.org/classifiers/ -classifiers = - Development Status :: 4 - Beta - Programming Language :: Python - - -[options] -zip_safe = False -include_package_data = True -package_dir = - =src - -# Require a min/specific Python version (comma-separated conditions) -# python_requires = >=3.8 - -# Add here dependencies of your project (line-separated), e.g. requests>=2.2,<3.0. -# Version specifiers like >=2.2,<3.0 avoid problems due to API changes in -# new major versions. This works if the required packages follow Semantic Versioning. -# For more information, check out https://semver.org/. -install_requires = - importlib-metadata; python_version<"3.8" - astropy - numpy - pandas - panoptes-utils[images] - pip - pydantic - pydantic-settings - python-dateutil - rich[jupyter] - tabulate - tqdm - typer - -[options.packages.find] -where = src -exclude = - tests - -[options.extras_require] -# Add here additional requirements for extra features, to install with: -# `pip install panoptes-data[PDF]` like: -# PDF = ReportLab; RXP - -# Add here test requirements (semicolon/line-separated) -testing = - setuptools - pytest - pytest-cov - tox - -[options.entry_points] -console_scripts = - panoptes-data = panoptes.data.utils.cli.main:app - -[tool:pytest] -# Specify command line options as you would do when invoking pytest directly. -# e.g. --cov-report html (or xml) for html/xml output or --junitxml junit.xml -# in order to write a coverage file that can be read by Jenkins. -# CAUTION: --cov flags may prohibit setting breakpoints while debugging. -# Comment those flags to avoid this pytest issue. -addopts = - --cov panoptes.data --cov-report term-missing - --verbose -norecursedirs = - dist - build - .tox -testpaths = tests -# Use pytest markers to select/deselect specific tests -# markers = -# slow: mark tests as slow (deselect with '-m "not slow"') -# system: mark end-to-end system tests - -[devpi:upload] -# Options for the devpi: PyPI server and packaging tool -# VCS export must be deactivated since we are using setuptools-scm -no_vcs = 1 -formats = bdist_wheel - -[flake8] -# Some sane defaults for the code style checker flake8 -max_line_length = 100 -extend_ignore = E203, W503 -# ^ Black-compatible -# E203 and W503 have edge cases handled by black -exclude = - .tox - build - dist - .eggs - docs/conf.py - -[pyscaffold] -# PyScaffold's parameters when the project was created. -# This will be used when updating. Do not change! -version = 4.5 -package = data -extensions = - github_actions - markdown - namespace - pre_commit -namespace = panoptes diff --git a/setup.py b/setup.py deleted file mode 100644 index 6dd5e6a..0000000 --- a/setup.py +++ /dev/null @@ -1,21 +0,0 @@ -""" - Setup file for panoptes-data. - Use setup.cfg to configure your project. - - This file was generated with PyScaffold 4.5. - PyScaffold helps you to put up the scaffold of your new Python project. - Learn more under: https://pyscaffold.org/ -""" -from setuptools import setup - -if __name__ == "__main__": - try: - setup(use_scm_version={"version_scheme": "no-guess-dev"}) - except: # noqa - print( - "\n\nAn error occurred while building the project, " - "please ensure you have the most updated version of setuptools, " - "setuptools_scm and wheel with:\n" - " pip install -U setuptools setuptools_scm wheel\n\n" - ) - raise diff --git a/src/panoptes/data/images.py b/src/panoptes/data/images.py deleted file mode 100644 index 11861e5..0000000 --- a/src/panoptes/data/images.py +++ /dev/null @@ -1,158 +0,0 @@ -from dataclasses import dataclass -from datetime import datetime -from enum import IntEnum, auto -from pathlib import Path -from typing import Union - -from astropy.time import Time -from dateutil.parser import parse as parse_date -from panoptes.utils.images import fits as fits_utils -from panoptes.utils.time import flatten_time - -from panoptes.data.settings import PATH_MATCHER - - -class ImageStatus(IntEnum): - """The status of an image.""" - ERROR = auto() - MASKED = auto() - UNKNOWN = auto() - RECEIVING = auto() - RECEIVED = auto() - UNSOLVED = auto() - PROCESSING = auto() - CALIBRATING = auto() - CALIBRATED = auto() - SOLVING = auto() - SOLVED = auto() - MATCHING = auto() - MATCHED = auto() - EXTRACTING = auto() - EXTRACTED = auto() - - -@dataclass -class ImagePathInfo: - """Parse the location path for an image. - - This is a small dataclass that offers some convenience methods for dealing - with a path based on the image id. - - This would usually be instantiated via `path`: - - ..doctest:: - - >>> from panoptes.data.images import ImagePathInfo - >>> bucket_path = 'gs://panoptes-images-background/PAN012/Hd189733/358d0f/20180824T035917/20180824T040118.fits' - >>> path_info = ImagePathInfo(path=bucket_path) - - >>> path_info.id - 'PAN012_358d0f_20180824T035917_20180824T040118' - - >>> path_info.unit_id - 'PAN012' - - >>> path_info.sequence_id - 'PAN012_358d0f_20180824T035917' - - >>> path_info.sequence_time -