Reporting CLI Scripts

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -1,14 +1,14 @@
-name: CWMS CLI Script Issues
+name: CWMS-CLI Script Issues
 description: File a bug report CWMS-CLI scripts
 labels: ["bug"]
 body:
-
   - type: dropdown
     id: script
     attributes:
       label: CLI Script
       description: Select the script this pertains to
       options:
+        - blob
         - cwms-cli
         - csv2cwms
         - getusgs-measurements

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -1,4 +1,4 @@
-name: CWMS-CLI Script Issue or Feature Request
+name: CWMS-CLI Feature Request
 description: Request a feature related to CWMS-CLI scripts
 labels: ["enhancement"]
 body:
@@ -8,6 +8,7 @@ body:
       label: CLI Script
       description: Select the script this pertains to
       options:
+        - blob
         - cwms-cli
         - csv2cwms
         - getusgs-measurements

.github/workflows/docs.yml

@@ -0,0 +1,25 @@
+# Confirm the docs build works on PRs touching docs or code
+name: ReadTheDocs/Sphinx Validation
+on:
+  pull_request:
+    paths: ["docs/**", "cwmscli/**", "pyproject.toml"]
+  push:
+    branches: [main]
+    paths: ["docs/**", "cwmscli/**", "pyproject.toml"]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6 
+        with:
+          python-version: "3.12"
+      - name: Install deps
+        run: |
+          python -m pip install -U pip
+          pip install -r docs/requirements.txt
+          pip install .
+      - name: Sphinx build (treat warnings as errors)
+        run: sphinx-build -nW -b html docs docs/_build/html
+      - name: Link check (optional)
+        run: sphinx-build -b linkcheck docs docs/_build/linkcheck

.readthedocs.yaml

@@ -0,0 +1,16 @@
+version: 2
+
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.12"
+
+sphinx:
+  configuration: docs/conf.py
+
+python:
+  install:
+    - method: pip
+      path: .
+    - requirements: docs/requirements.txt
+formats: [pdf]

README.md

@@ -1,13 +1,17 @@
 # cwms-cli
+
 command line utilities used for Corps Water Management Systems (CWMS) processes
 
+[![Docs](https://readthedocs.org/projects/cwms-cli/badge/?version=latest)](https://cwms-cli.readthedocs.io/en/latest/)
+
 ## Install
+
 ```sh
 pip install git+https://github.com/HydrologicEngineeringCenter/cwms-cli.git@main
 ```
 
 ## Command line implementation
+
 ```sh
 cwms-cli --help
 ```
-

cwmscli/__main__.py

@@ -1,18 +1,17 @@
 import click
 
 from cwmscli.commands import commands_cwms
-from cwmscli.getusgs import commands_getusgs
+from cwmscli.reporting import reporting_cli
+from cwmscli.usgs import usgs_group
 
 
 @click.group()
 def cli():
     pass
 
 
-cli.add_command(commands_getusgs.getusgs_timeseries)
-cli.add_command(commands_getusgs.getusgs_ratings)
-cli.add_command(commands_getusgs.ratingsinifileimport)
-cli.add_command(commands_getusgs.getusgs_measurements)
+cli.add_command(usgs_group, name="usgs")
 cli.add_command(commands_cwms.shefcritimport)
 cli.add_command(commands_cwms.csv2cwms_cmd)
 cli.add_command(commands_cwms.blob_group)
+cli.add_command(reporting_cli)

cwmscli/commands/blob.py

@@ -1,5 +1,4 @@
 import base64
-import imghdr
 import json
 import logging
 import mimetypes
@@ -13,11 +12,42 @@
 import requests
 
 from cwmscli.utils import get_api_key
+from cwmscli.utils.deps import requires
 
 # used to rebuild data URL for images
 DATA_URL_RE = re.compile(r"^data:(?P<mime>[^;]+);base64,(?P<data>.+)$", re.I | re.S)
 
 
+@requires(
+    {
+        "module": "imghdr",
+        "package": "standard-imghdr",
+        "version": "3.0.0",
+        "desc": "Package to help detect image types",
+        "link": "https://docs.python.org/3/library/imghdr.html",
+    }
+)
+def _determine_ext(data: bytes | str, write_type: str) -> str:
+    """
+    Attempt to determine the file extension from the data itself.
+    Requires the imghdr module (lazy import) to inspect the bytes for image types.
+    If not an image, defaults to .bin
+
+    Args:
+        data: The binary data or base64 string to inspect.
+        write_type: The mode in which the data will be written ('wb' for binary, 'w' for text).
+
+    Returns:
+        The determined file extension, including the leading dot (e.g., '.png', '.jpg').
+    """
+    import imghdr
+
+    kind = imghdr.what(None, data)
+    if kind == "jpeg":
+        kind = "jpg"
+    return f".{kind}" if kind else ".bin"
+
+
 def _save_base64(
     b64_or_dataurl: str,
     dest: str,
@@ -48,11 +78,10 @@ def _save_base64(
             ext = mimetypes.guess_extension(media_type.split(";")[0].lower()) or ""
             if ext == ".jpe":
                 ext = ".jpg"
+        # last resort, try to determine from the data itself
+        # requires imghdr to dig into the bytes to determine image type
         if not ext:
-            kind = imghdr.what(None, data)
-            if kind == "jpeg":
-                kind = "jpg"
-            ext = f".{kind}" if kind else ".bin"
+            ext = _determine_ext(data, write_type)
         dest = base + ext
 
     os.makedirs(os.path.dirname(dest) or ".", exist_ok=True)

cwmscli/commands/commands_cwms.py

@@ -36,11 +36,11 @@ def shefcritimport(filename, office, api_root, api_key, api_key_loc):
 @click.command("csv2cwms", help="Store CSV TimeSeries data to CWMS using a config file")
 @common_api_options
 @click.option(
-    "-l",
-    "--location",
+    "--input-keys",
+    "input_keys",
     default="all",
     show_default=True,
-    help='Location ID. Use "-p=all" for all locations.',
+    help='Input keys. Defaults to all keys/files with --input-keys=all. These are the keys under "input_files" in a given config file. This option lets you run a single file from a config that contains multiple files. Example: --input-keys=file1',
 )
 @click.option(
     "-lb",
@@ -67,15 +67,6 @@ def shefcritimport(filename, office, api_root, api_key, api_key_loc):
     help="Override CSV file (else use config)",
 )
 @click.option("--log", show_default=True, help="Path to the log file.")
-@click.option(
-    "-dp",
-    "--data-path",
-    "data_path",
-    default=".",
-    show_default=True,
-    type=click.Path(exists=True, file_okay=False),
-    help="Directory where csv files are stored",
-)
 @click.option("--dry-run", is_flag=True, help="Log only (no HTTP calls)")
 @click.option("--begin", type=str, help="YYYY-MM-DDTHH:MM (local to --tz)")
 @click.option("-tz", "--timezone", "tz", default="GMT", show_default=True)
@@ -100,7 +91,16 @@ def csv2cwms_cmd(**kwargs):
 # ================================================================================
 #  BLOB
 # ================================================================================
-@click.group("blob", help="Manage CWMS Blobs (upload, download, delete, update, list)")
+@click.group(
+    "blob",
+    help="Manage CWMS Blobs (upload, download, delete, update, list)",
+    epilog="""
+  * Store a PDF/image as a CWMS blob with optional description
+  * Download a blob by id to your local filesystem
+  * Update a blob's name/description
+  * Bulk list blobs for an office
+""",
+)
 @requires(reqs.cwms)
 def blob_group():
     pass
@@ -190,7 +190,9 @@ def update_cmd(**kwargs):
 # ================================================================================
 @blob_group.command("list", help="List blobs with optional filters and sorting")
 # TODO: Add link to regex docs when new CWMS-DATA site is deployed to PROD
-@click.option("--blob-id-like", help="LIKE filter for blob ID (e.g., '*PNG').")
+@click.option(
+    "--blob-id-like", help="LIKE filter for blob ID (e.g., ``*PNG``)."
+)  # Escape the wildcard/asterisk for RTD generation with double backticks
 @click.option(
     "--columns",
     multiple=True,

cwmscli/commands/csv2cwms/README.md

@@ -8,36 +8,44 @@ To View the Help: `cwms-cli csv2cwms --help`
 
 Usage: cwms-cli csv2cwms [OPTIONS]
 
-  Store CSV TimeSeries data to CWMS using a config file
+Store CSV TimeSeries data to CWMS using a config file
 
 Options:
-  -o, --office TEXT           Office to grab data for  [required]
-  -a, --api_root TEXT         Api Root for CDA. Can be user defined or placed
-                              in a env variable CDA_API_ROOT  [required]
-  -k, --api_key TEXT          api key for CDA. Can be user defined or place in
-                              env variable CDA_API_KEY. one of api_key or
-                              api_key_loc are required
-  -l, --location TEXT         Location ID. Use "-p=all" for all locations.
-                              [default: all]
-  -lb, --lookback INTEGER     Lookback period in HOURS  [default: 120]
-  -v, --verbose               Verbose logging
-  -c, --config PATH           Path to JSON config file  [required]
-                              [default: all]
-  -lb, --lookback INTEGER     Lookback period in HOURS  [default: 120]
-  -v, --verbose               Verbose logging
-                              [default: all]
-                              [default: all]
-  -lb, --lookback INTEGER     Lookback period in HOURS  [default: 120]
-  -v, --verbose               Verbose logging
-  -c, --config PATH           Path to JSON config file  [required]
-  -df, --data-file TEXT       Override CSV file (else use config)
-  --log TEXT                  Path to the log file.
-  -dp, --data-path DIRECTORY  Directory where csv files are stored  [default:
-                              .]
-  --dry-run                   Log only (no HTTP calls)
-  --begin TEXT                YYYY-MM-DDTHH:MM (local to --tz)
-  -tz, --timezone TEXT        [default: GMT]
-  --ignore-ssl-errors         Ignore TLS errors (testing only)
-  --version                   Show the version and exit.
-  --help                      Show this message and exit.
+-o, --office TEXT Office to grab data for [required]
+-a, --api_root TEXT Api Root for CDA. Can be user defined or placed
+in a env variable CDA_API_ROOT [required]
+-k, --api_key TEXT api key for CDA. Can be user defined or place in
+env variable CDA_API_KEY. one of api_key or
+api_key_loc are required
+-l, --location TEXT Location ID. Use "-p=all" for all locations.
+[default: all]
+-lb, --lookback INTEGER Lookback period in HOURS [default: 120]
+-v, --verbose Verbose logging
+-c, --config PATH Path to JSON config file [required]
+[default: all]
+-lb, --lookback INTEGER Lookback period in HOURS [default: 120]
+-v, --verbose Verbose logging
+[default: all]
+[default: all]
+-lb, --lookback INTEGER Lookback period in HOURS [default: 120]
+-v, --verbose Verbose logging
+-c, --config PATH Path to JSON config file [required]
+-df, --data-file TEXT Override CSV file (else use config)
+--log TEXT Path to the log file.
+-dp, --data-path DIRECTORY Directory where csv files are stored [default:
+.]
+--dry-run Log only (no HTTP calls)
+--begin TEXT YYYY-MM-DDTHH:MM (local to --tz)
+-tz, --timezone TEXT [default: GMT]
+--ignore-ssl-errors Ignore TLS errors (testing only)
+--version Show the version and exit.
+--help Show this message and exit.
 
+## Features
+
+- Allow for specifying one or more date formats that might be seen per input csv file
+- Allow mathematical operations across multiple columns and storing into one timeseries
+- Store one column of data with a user-specified precision and units to a timeseries identifier
+- Dry runs to test what data might look like prior to database storage
+- Verbose logging via the -v flag
+- Colored terminal output for user readability