diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 455585f..8988cb2 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,25 +1,25 @@ -# Confirm the docs build works on PRs touching docs or code -name: ReadTheDocs/Sphinx Validation -on: - pull_request: - paths: ["docs/**", "cwmscli/**", "pyproject.toml"] - push: - branches: [main] - paths: ["docs/**", "cwmscli/**", "pyproject.toml"] -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: "3.12" - - name: Install deps - run: | - python -m pip install -U pip - pip install -r docs/requirements.txt - pip install . - - name: Sphinx build (treat warnings as errors) - run: sphinx-build -nW -b html docs docs/_build/html - - name: Link check (optional) - run: sphinx-build -b linkcheck docs docs/_build/linkcheck +# Confirm the docs build works on PRs touching docs or code +name: ReadTheDocs/Sphinx Validation +on: + pull_request: + paths: ["docs/**", "cwmscli/**", "pyproject.toml"] + push: + branches: [main] + paths: ["docs/**", "cwmscli/**", "pyproject.toml"] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install deps + run: | + python -m pip install -U pip + pip install -r docs/requirements.txt + pip install . + - name: Sphinx build (treat warnings as errors) + run: sphinx-build -nW -b html docs docs/_build/html + - name: Link check (optional) + run: sphinx-build -b linkcheck docs docs/_build/linkcheck \ No newline at end of file diff --git a/cwmscli/__main__.py b/cwmscli/__main__.py index ba12def..678a1eb 100644 --- a/cwmscli/__main__.py +++ b/cwmscli/__main__.py @@ -13,3 +13,4 @@ def cli(): cli.add_command(commands_cwms.shefcritimport) cli.add_command(commands_cwms.csv2cwms_cmd) cli.add_command(commands_cwms.blob_group) +cli.add_command(commands_cwms.clob_group) diff --git a/cwmscli/commands/blob.py b/cwmscli/commands/blob.py index 2fdebf2..7db1b3c 100644 --- a/cwmscli/commands/blob.py +++ b/cwmscli/commands/blob.py @@ -5,13 +5,13 @@ import os import re import sys -from typing import Optional, Sequence +from typing import Optional, Sequence, Union import cwms import pandas as pd import requests -from cwmscli.utils import get_api_key +from cwmscli.utils import get_api_key, has_invalid_chars from cwmscli.utils.deps import requires # used to rebuild data URL for images @@ -89,107 +89,6 @@ def _save_base64( return dest -def store_blob(**kwargs): - file_data = kwargs.get("file_data") - blob_id = kwargs.get("blob_id", "").upper() - # Attempt to determine what media type should be used for the mime-type if one is not presented based on the file extension - media = kwargs.get("media_type") or get_media_type(kwargs.get("input_file")) - - logging.debug( - f"Office: {kwargs.get('office')} Output ID: {blob_id} Media: {media}" - ) - - blob = { - "office-id": kwargs.get("office"), - "id": blob_id, - "description": json.dumps(kwargs.get("description")), - "media-type-id": media, - "value": base64.b64encode(file_data).decode("utf-8"), - } - - params = {"fail-if-exists": not kwargs.get("overwrite")} - - if kwargs.get("dry_run"): - logging.info( - f"--dry-run enabled. Would POST to {kwargs.get('api_root')}/blobs with params={params}" - ) - logging.info( - f"Blob payload summary: office-id={kwargs.get('office')}, id={blob_id}, media={media}", - ) - logging.info( - json.dumps( - { - "url": f"{kwargs.get('api_root')}blobs", - "params": params, - "blob": {**blob, "value": f""}, - }, - indent=2, - ) - ) - sys.exit(0) - - try: - cwms.store_blobs(blob, fail_if_exists=kwargs.get("overwrite")) - logging.info(f"Successfully stored blob with ID: {blob_id}") - logging.info( - f"View: {kwargs.get('api_root')}blobs/{blob_id}?office={kwargs.get('office')}" - ) - except requests.HTTPError as e: - # Include response text when available - detail = getattr(e.response, "text", "") or str(e) - logging.error(f"Failed to store blob (HTTP): {detail}") - sys.exit(1) - except Exception as e: - logging.error(f"Failed to store blob: {e}") - sys.exit(1) - - -def retrieve_blob(**kwargs): - blob_id = kwargs.get("blob_id", "").upper() - if not blob_id: - logging.warning( - "Valid blob_id required to download a blob. cwms-cli blob download --blob-id=myid. Run the list directive to see options for your office." - ) - sys.exit(0) - logging.debug(f"Office: {kwargs.get('office')} Blob ID: {blob_id}") - try: - blob = cwms.get_blob( - office_id=kwargs.get("office"), - blob_id=blob_id, - ) - logging.info( - f"Successfully retrieved blob with ID: {blob_id}", - ) - _save_base64(blob, dest=blob_id) - logging.info(f"Downloaded blob to: {blob_id}") - except requests.HTTPError as e: - detail = getattr(e.response, "text", "") or str(e) - logging.error(f"Failed to retrieve blob (HTTP): {detail}") - sys.exit(1) - except Exception as e: - logging.error(f"Failed to retrieve blob: {e}") - sys.exit(1) - - -def delete_blob(**kwargs): - blob_id = kwargs.get("blob_id").upper() - logging.debug(f"Office: {kwargs.get('office')} Blob ID: {blob_id}") - - try: - # cwms.delete_blob( - # office_id=kwargs.get("office"), - # blob_id=kwargs.get("blob_id").upper(), - # ) - logging.info(f"Successfully deleted blob with ID: {blob_id}") - except requests.HTTPError as e: - details = getattr(e.response, "text", "") or str(e) - logging.error(f"Failed to delete blob (HTTP): {details}") - sys.exit(1) - except Exception as e: - logging.error(f"Failed to delete blob: {e}") - sys.exit(1) - - def list_blobs( office: Optional[str] = None, blob_id_like: Optional[str] = None, @@ -294,7 +193,12 @@ def upload_cmd( try: cwms.store_blobs(blob, fail_if_exists=not overwrite) logging.info(f"Uploaded blob: {blob_id_up}") - logging.info(f"View: {api_root}blobs/{blob_id_up}?office={office}") + if has_invalid_chars(blob_id_up): + logging.info( + f"View: {api_root}blobs/ignored?blob-id={blob_id_up}&office={office}" + ) + else: + logging.info(f"View: {api_root}blobs/{blob_id_up}?office={office}") except requests.HTTPError as e: detail = getattr(e.response, "text", "") or str(e) logging.error(f"Failed to upload (HTTP): {detail}") @@ -414,4 +318,12 @@ def list_cmd( else: # Friendly console preview with pd.option_context("display.max_rows", 500, "display.max_columns", None): - logging.info(df.to_string(index=False)) + # Left-align all columns + logging.info( + "\n" + + df.apply( + lambda s: (s := s.astype(str).str.strip()).str.ljust( + s.str.len().max() + ) + ).to_string(index=False, justify="left") + ) diff --git a/cwmscli/commands/clob.py b/cwmscli/commands/clob.py new file mode 100644 index 0000000..1ce5b7b --- /dev/null +++ b/cwmscli/commands/clob.py @@ -0,0 +1,241 @@ +import base64 +import json +import logging +import mimetypes +import os +import re +import sys +from typing import Optional, Sequence + +import cwms +import pandas as pd +import requests + +from cwmscli.utils import get_api_key, has_invalid_chars + + +def list_clobs( + office: Optional[str] = None, + clob_id_like: Optional[str] = None, + columns: Optional[Sequence[str]] = None, + sort_by: Optional[Sequence[str]] = None, + ascending: bool = True, + limit: Optional[int] = None, +) -> pd.DataFrame: + logging.info(f"Listing clobs for office: {office!r}...") + result = cwms.get_clobs(office_id=office, clob_id_like=clob_id_like) + + # Accept either a DataFrame or a JSON/dict-like response + if isinstance(result, pd.DataFrame): + df = result.copy() + else: + # Expecting normal clob return structure + data = getattr(result, "json", None) + if callable(data): + data = result.json() + df = pd.DataFrame((data or {}).get("clobs", [])) + + # Allow column filtering + if columns: + keep = [c for c in columns if c in df.columns] + if keep: + df = df[keep] + + # Sort by option + if sort_by: + by = [c for c in sort_by if c in df.columns] + if by: + df = df.sort_values(by=by, ascending=ascending, kind="stable") + + # Optional limit + if limit is not None: + df = df.head(limit) + + logging.info(f"Found {len(df):,} clob(s)") + # List the clobs in the logger + for _, row in df.iterrows(): + logging.info(f"clob ID: {row['id']}, Description: {row.get('description')}") + return df + + +def upload_cmd( + input_file: str, + clob_id: str, + description: str, + overwrite: bool, + dry_run: bool, + office: str, + api_root: str, + api_key: str, +): + cwms.init_session(api_root=api_root, api_key=get_api_key(api_key, "")) + try: + file_size = os.path.getsize(input_file) + with open(input_file, "r") as f: + file_data = f.read() + logging.info(f"Read file: {input_file} ({file_size} bytes)") + except Exception as e: + logging.error(f"Failed to read file: {e}") + sys.exit(1) + + clob_id_up = clob_id.upper() + logging.debug(f"Office={office} clobID={clob_id_up}") + + clob = { + "office-id": office, + "id": clob_id_up, + "description": ( + json.dumps(description) + if isinstance(description, (dict, list)) + else description + ), + "value": file_data, + } + params = {"fail-if-exists": not overwrite} + + if dry_run: + logging.info(f"DRY RUN: would POST {api_root}clobs with params={params}") + logging.info( + json.dumps( + { + "url": f"{api_root}clobs", + "params": params, + "clob": {**clob, "value": f'<{len(clob["value"])} chars>'}, + }, + indent=2, + ) + ) + return + + try: + cwms.store_clobs(clob, fail_if_exists=not overwrite) + logging.info(f"Uploaded clob: {clob_id_up}") + # IDs with / can't be used directly in the path + # TODO: check for other disallowed characters + if has_invalid_chars(clob_id_up): + logging.info( + f"View: {api_root}clobs/ignored?clob-id={clob_id_up}&office={office}" + ) + else: + logging.info(f"View: {api_root}clobs/{clob_id_up}?office={office}") + except requests.HTTPError as e: + detail = getattr(e.response, "text", "") or str(e) + logging.error(f"Failed to upload (HTTP): {detail}") + sys.exit(1) + except Exception as e: + logging.error(f"Failed to upload: {e}") + sys.exit(1) + + +def download_cmd( + clob_id: str, dest: str, office: str, api_root: str, api_key: str, dry_run: bool +): + if dry_run: + logging.info( + f"DRY RUN: would GET {api_root} clob with clob-id={clob_id} office={office}." + ) + return + cwms.init_session(api_root=api_root, api_key=get_api_key(api_key, "")) + bid = clob_id.upper() + logging.debug(f"Office={office} clobID={bid}") + + try: + clob = cwms.get_clob(office_id=office, clob_id=bid) + os.makedirs(os.path.dirname(dest) or ".", exist_ok=True) + sys.stderr.write(repr(clob.json) + "\n") + with open(dest, "wt") as f: + f.write(clob.json["value"]) + + logging.info(f"Downloaded clob to: {dest}") + except requests.HTTPError as e: + detail = getattr(e.response, "text", "") or str(e) + logging.error(f"Failed to download (HTTP): {detail}") + sys.exit(1) + except Exception as e: + logging.error(f"Failed to download: {e}") + sys.exit(1) + + +def delete_cmd(clob_id: str, office: str, api_root: str, api_key: str, dry_run: bool): + + if dry_run: + logging.info( + f"DRY RUN: would DELETE {api_root} clob with clob-id={clob_id} office={office}" + ) + return + cwms.init_session(api_root=api_root, api_key=api_key) + cwms.delete_clob(office_id=office, clob_id=clob_id) + logging.info(f"Deleted clob: {clob_id} for office: {office}") + + +def update_cmd( + input_file: str, + clob_id: str, + description: str, + ignore_nulls: bool, + dry_run: bool, + office: str, + api_root: str, + api_key: str, +): + if dry_run: + logging.info( + f"DRY RUN: would PATCH {api_root} clob with clob-id={clob_id} office={office}" + ) + return + file_data = None + if input_file: + try: + file_size = os.path.getsize(input_file) + with open(input_file, "r") as f: + file_data = f.read() + logging.info(f"Read file: {input_file} ({file_size} bytes)") + except Exception as e: + logging.error(f"Failed to read file: {e}") + sys.exit(1) + # Setup minimum required payload + clob = {"office-id": office, "id": clob_id.upper()} + if description: + clob["description"] = description + + if file_data: + clob["value"] = file_data + cwms.init_session(api_root=api_root, api_key=api_key) + cwms.update_clob(clob, clob_id.upper(), ignore_nulls=ignore_nulls) + + +def list_cmd( + clob_id_like: str, + columns: list[str], + sort_by: list[str], + desc: bool, + limit: int, + to_csv: str, + office: str, + api_root: str, + api_key: str, +): + cwms.init_session(api_root=api_root, api_key=get_api_key(api_key, None)) + df = list_clobs( + office=office, + clob_id_like=clob_id_like, + columns=columns, + sort_by=sort_by, + ascending=not desc, + limit=limit, + ) + if to_csv: + df.to_csv(to_csv, index=False) + logging.info(f"Wrote {len(df)} rows to {to_csv}") + else: + # Friendly console preview + with pd.option_context("display.max_rows", 500, "display.max_columns", None): + # Left-align all columns + logging.info( + "\n" + + df.apply( + lambda s: (s := s.astype(str).str.strip()).str.ljust( + s.str.len().max() + ) + ).to_string(index=False, justify="left") + ) diff --git a/cwmscli/commands/commands_cwms.py b/cwmscli/commands/commands_cwms.py index a7b63a6..be6c675 100644 --- a/cwmscli/commands/commands_cwms.py +++ b/cwmscli/commands/commands_cwms.py @@ -250,3 +250,154 @@ def list_cmd(**kwargs): # endregion + + +# region Clob +# ================================================================================ +# CLOB +# ================================================================================ +@click.group( + "clob", + help="Manage CWMS Clobs (upload, download, delete, update, list)", + epilog=textwrap.dedent( + """ + Example Usage:\n + - Download a clob by id to your local filesystem\n + - Update a clob's name/description/mime-type\n + - Bulk list clobs for an office +""" + ), +) +@requires(reqs.cwms) +def clob_group(): + pass + + +# ================================================================================ +# Upload +# ================================================================================ +@clob_group.command("upload", help="Upload a file as a clob") +@click.option( + "--input-file", + required=True, + type=click.Path(exists=True, dir_okay=False, readable=True, path_type=str), + help="Path to the file to upload.", +) +@click.option("--clob-id", required=True, type=str, help="Clob ID to create.") +@click.option("--description", default=None, help="Optional description JSON or text.") +@click.option( + "--overwrite/--no-overwrite", + default=False, + show_default=True, + help="If true, replace existing clob.", +) +@click.option("--dry-run", is_flag=True, help="Show request; do not send.") +@common_api_options +def clob_upload(**kwargs): + from cwmscli.commands.clob import upload_cmd + + upload_cmd(**kwargs) + + +# ================================================================================ +# Download +# ================================================================================ +@clob_group.command("download", help="Download a clob by ID") +# TODO: test XML +@click.option("--clob-id", required=True, type=str, help="Clob ID to download.") +@click.option( + "--dest", + default=None, + help="Destination file path. Defaults to clob-id.", +) +@click.option("--dry-run", is_flag=True, help="Show request; do not send.") +@common_api_options +def clob_download(**kwargs): + from cwmscli.commands.clob import download_cmd + + download_cmd(**kwargs) + + +# ================================================================================ +# Delete +# ================================================================================ +@clob_group.command("delete", help="Delete a clob by ID") +@click.option("--clob-id", required=True, type=str, help="Clob ID to delete.") +@click.option("--dry-run", is_flag=True, help="Show request; do not send.") +@common_api_options +def delete_cmd(**kwargs): + from cwmscli.commands.clob import delete_cmd + + delete_cmd(**kwargs) + + +# ================================================================================ +# Update +# ================================================================================ +@clob_group.command("update", help="Update/patch a clob by ID") +@click.option("--clob-id", required=True, type=str, help="Clob ID to update.") +@click.option("--dry-run", is_flag=True, help="Show request; do not send.") +@click.option( + "--description", + default=None, + help="New description JSON or text.", +) +@click.option( + "--input-file", + required=False, + type=click.Path(exists=True, dir_okay=False, readable=True, path_type=str), + help="Optional file content to upload with update.", +) +@click.option( + "--ignore-nulls/--no-ignore-nulls", + default=True, + show_default=True, + help="If true, null and empty fields in the provided clob will be ignored and the existing value of those fields left in place.", +) +@common_api_options +def update_cmd(**kwargs): + from cwmscli.commands.clob import update_cmd + + update_cmd(**kwargs) + + +# ================================================================================ +# List +# ================================================================================ +@clob_group.command("list", help="List clobs with optional filters and sorting") +# TODO: Add link to regex docs when new CWMS-DATA site is deployed to PROD +@click.option( + "--clob-id-like", help="LIKE filter for clob ID (e.g., ``*PNG``)." +) # Escape the wildcard/asterisk for RTD generation with double backticks +@click.option( + "--columns", + multiple=True, + callback=csv_to_list, + help="Columns to show (repeat or comma-separate).", +) +@click.option( + "--sort-by", + multiple=True, + callback=csv_to_list, + help="Columns to sort by (repeat or comma-separate).", +) +@click.option( + "--desc/--asc", + default=False, + show_default=True, + help="Sort descending instead of ascending.", +) +@click.option("--limit", type=int, default=None, help="Max rows to show.") +@click.option( + "--to-csv", + type=click.Path(dir_okay=False, writable=True, path_type=str), + help="If set, write results to this CSV file.", +) +@common_api_options +def list_cmd(**kwargs): + from cwmscli.commands.clob import list_cmd + + list_cmd(**kwargs) + + +# endregion diff --git a/cwmscli/utils/__init__.py b/cwmscli/utils/__init__.py index 7f0f41b..6e5cca5 100644 --- a/cwmscli/utils/__init__.py +++ b/cwmscli/utils/__init__.py @@ -7,6 +7,18 @@ def to_uppercase(ctx, param, value): return value.upper() +def has_invalid_chars(id: str) -> bool: + """ + Checks if ID contains any invalid web path characters. + """ + INVALID_PATH_CHARS = ["/", "\\", "&", "?", "="] + + for char in INVALID_PATH_CHARS: + if char in id: + return True + return False + + office_option = click.option( "-o", "--office",