From 705f820429a3603c2352c5fd56d09f7fbf214a95 Mon Sep 17 00:00:00 2001 From: Charles Graham SWT Date: Tue, 16 Sep 2025 16:29:39 -0500 Subject: [PATCH 01/25] Add yaml for config files --- poetry.lock | 63 ++++++++++++++++++++++++++++++++++++++++++-------- pyproject.toml | 1 + 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/poetry.lock b/poetry.lock index 487d75b..692853c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. [[package]] name = "black" @@ -6,6 +6,7 @@ version = "24.10.0" description = "The uncompromising code formatter." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "black-24.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6668650ea4b685440857138e5fe40cde4d652633b1bdffc62933d0db4ed9812"}, {file = "black-24.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1c536fcf674217e87b8cc3657b81809d3c085d7bf3ef262ead700da345bfa6ea"}, @@ -52,6 +53,7 @@ version = "2025.8.3" description = "Python package for providing Mozilla's CA Bundle." optional = true python-versions = ">=3.7" +groups = ["main"] files = [ {file = "certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5"}, {file = "certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407"}, @@ -63,6 +65,7 @@ version = "3.4.0" description = "Validate configuration and produce human readable error messages." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"}, {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, @@ -74,6 +77,7 @@ version = "3.4.3" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = true python-versions = ">=3.7" +groups = ["main"] files = [ {file = "charset_normalizer-3.4.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72"}, {file = "charset_normalizer-3.4.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe"}, @@ -162,6 +166,7 @@ version = "8.1.8" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, @@ -176,6 +181,8 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "dev"] +markers = "platform_system == \"Windows\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -187,6 +194,7 @@ version = "0.8.0" description = "Corps water management systems (CWMS) REST API for Data Retrieval of USACE water data" optional = true python-versions = "<4.0,>=3.9" +groups = ["main"] files = [ {file = "cwms_python-0.8.0-py3-none-any.whl", hash = "sha256:2ab7f6b6ca54a8f3e8c8a1421eefed008f96cf2b81772bbce82422bf154c173f"}, {file = "cwms_python-0.8.0.tar.gz", hash = "sha256:71da687f35680ddb88bdba0464eb3585d05289f507872fdcd31e85483f26654b"}, @@ -203,6 +211,7 @@ version = "0.4.0" description = "Distribution utilities" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16"}, {file = "distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d"}, @@ -214,6 +223,7 @@ version = "1.9.0" description = "Distro - an OS platform information API" optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2"}, {file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"}, @@ -225,6 +235,7 @@ version = "3.19.1" description = "A platform independent file lock." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d"}, {file = "filelock-3.19.1.tar.gz", hash = "sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58"}, @@ -236,6 +247,7 @@ version = "0.1.25" description = "Python wrapper for the HEC-DSS file database C library." optional = true python-versions = ">=3.8" +groups = ["main"] files = [ {file = "hecdss-0.1.25-py3-none-any.whl", hash = "sha256:26ac388e220f197e7105c1f1264d704d3afd4d6ba156d58cfe7d2e7b5ed420cb"}, {file = "hecdss-0.1.25.tar.gz", hash = "sha256:19086abc7111932a19719afa7caadaaeacd80f500ad2d5310cb9641d21d0c5d1"}, @@ -251,6 +263,7 @@ version = "2.6.13" description = "File identification library for Python" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "identify-2.6.13-py2.py3-none-any.whl", hash = "sha256:60381139b3ae39447482ecc406944190f690d4a2997f2584062089848361b33b"}, {file = "identify-2.6.13.tar.gz", hash = "sha256:da8d6c828e773620e13bfa86ea601c5a5310ba4bcd65edf378198b56a1f9fb32"}, @@ -265,6 +278,7 @@ version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = true python-versions = ">=3.6" +groups = ["main"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -279,6 +293,7 @@ version = "5.13.2" description = "A Python utility / library to sort Python imports." optional = false python-versions = ">=3.8.0" +groups = ["dev"] files = [ {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, @@ -293,6 +308,7 @@ version = "1.4.2" description = "Read settings from config files" optional = false python-versions = ">=3.7.1,<4.0.0" +groups = ["dev"] files = [ {file = "maison-1.4.2-py3-none-any.whl", hash = "sha256:b63fe6751494935fc453dfb76319af223e4cb8bab32ac5464c2a9ca0edda8765"}, {file = "maison-1.4.2.tar.gz", hash = "sha256:d2abac30a5c6a0749526d70ae95a63c6acf43461a1c10e51410b36734e053ec7"}, @@ -309,6 +325,7 @@ version = "1.17.1" description = "Optional static typing for Python" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "mypy-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3fbe6d5555bf608c47203baa3e72dbc6ec9965b3d7c318aa9a4ca76f465bd972"}, {file = "mypy-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80ef5c058b7bce08c83cac668158cb7edea692e458d21098c7d3bce35a5d43e7"}, @@ -369,6 +386,7 @@ version = "1.1.0" description = "Type system extensions for programs checked with the mypy type checker." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505"}, {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, @@ -380,6 +398,7 @@ version = "1.9.1" description = "Node.js virtual environment builder" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["dev"] files = [ {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"}, {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, @@ -391,6 +410,8 @@ version = "2.0.2" description = "Fundamental package for array computing in Python" optional = true python-versions = ">=3.9" +groups = ["main"] +markers = "python_version < \"3.11\"" files = [ {file = "numpy-2.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:51129a29dbe56f9ca83438b706e2e69a39892b5eda6cedcb6b0c9fdc9b0d3ece"}, {file = "numpy-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f15975dfec0cf2239224d80e32c3170b1d168335eaedee69da84fbe9f1f9cd04"}, @@ -445,6 +466,8 @@ version = "2.3.2" description = "Fundamental package for array computing in Python" optional = true python-versions = ">=3.11" +groups = ["main"] +markers = "python_version >= \"3.11\"" files = [ {file = "numpy-2.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:852ae5bed3478b92f093e30f785c98e0cb62fa0a939ed057c31716e18a7a22b9"}, {file = "numpy-2.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7a0e27186e781a69959d0230dd9909b5e26024f8da10683bd6344baea1885168"}, @@ -528,6 +551,7 @@ version = "25.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, @@ -539,6 +563,7 @@ version = "2.3.2" description = "Powerful data structures for data analysis, time series, and statistics" optional = true python-versions = ">=3.9" +groups = ["main"] files = [ {file = "pandas-2.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:52bc29a946304c360561974c6542d1dd628ddafa69134a7131fdfd6a5d7a1a35"}, {file = "pandas-2.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:220cc5c35ffaa764dd5bb17cf42df283b5cb7fdf49e10a7b053a06c9cb48ee2b"}, @@ -625,6 +650,7 @@ version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, @@ -636,6 +662,7 @@ version = "4.3.8" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4"}, {file = "platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc"}, @@ -652,6 +679,7 @@ version = "3.8.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pre_commit-3.8.0-py2.py3-none-any.whl", hash = "sha256:9a90a53bf82fdd8778d58085faf8d83df56e40dfe18f45b19446e26bf1b3a63f"}, {file = "pre_commit-3.8.0.tar.gz", hash = "sha256:8bb6494d4a20423842e198980c9ecf9f96607a07ea29549e180eef9ae80fe7af"}, @@ -670,6 +698,7 @@ version = "1.10.22" description = "Data validation and settings management using python type hints" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "pydantic-1.10.22-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:57889565ccc1e5b7b73343329bbe6198ebc472e3ee874af2fa1865cfe7048228"}, {file = "pydantic-1.10.22-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:90729e22426de79bc6a3526b4c45ec4400caf0d4f10d7181ba7f12c01bb3897d"}, @@ -736,6 +765,7 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -750,6 +780,7 @@ version = "2025.2" description = "World timezone definitions, modern and historical" optional = true python-versions = "*" +groups = ["main"] files = [ {file = "pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00"}, {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"}, @@ -761,6 +792,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -823,6 +855,7 @@ version = "2.32.5" description = "Python HTTP for Humans." optional = true python-versions = ">=3.9" +groups = ["main"] files = [ {file = "requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6"}, {file = "requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf"}, @@ -844,6 +877,7 @@ version = "1.0.0" description = "A utility belt for advanced users of python-requests" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main"] files = [ {file = "requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6"}, {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"}, @@ -858,6 +892,7 @@ version = "0.91.0" description = "ruyaml is a fork of ruamel.yaml" optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "ruyaml-0.91.0-py3-none-any.whl", hash = "sha256:50e0ee3389c77ad340e209472e0effd41ae0275246df00cdad0a067532171755"}, {file = "ruyaml-0.91.0.tar.gz", hash = "sha256:6ce9de9f4d082d696d3bde264664d1bcdca8f5a9dff9d1a1f1a127969ab871ab"}, @@ -876,19 +911,20 @@ version = "80.9.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922"}, {file = "setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.8.0)"] -core = ["importlib_metadata (>=6)", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] [[package]] name = "six" @@ -896,6 +932,7 @@ version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -907,6 +944,7 @@ version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +groups = ["dev"] files = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, @@ -918,6 +956,8 @@ version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" +groups = ["dev"] +markers = "python_version < \"3.11\"" files = [ {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, @@ -959,6 +999,7 @@ version = "4.14.1" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76"}, {file = "typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36"}, @@ -970,6 +1011,7 @@ version = "2025.2" description = "Provider of IANA time zone data" optional = true python-versions = ">=2" +groups = ["main"] files = [ {file = "tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8"}, {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, @@ -981,13 +1023,14 @@ version = "2.5.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = true python-versions = ">=3.9" +groups = ["main"] files = [ {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] @@ -998,6 +1041,7 @@ version = "20.34.0" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "virtualenv-20.34.0-py3-none-any.whl", hash = "sha256:341f5afa7eee943e4984a9207c025feedd768baff6753cd660c857ceb3e36026"}, {file = "virtualenv-20.34.0.tar.gz", hash = "sha256:44815b2c9dee7ed86e387b842a84f20b93f7f417f95886ca1996a72a4138eb1a"}, @@ -1011,7 +1055,7 @@ typing-extensions = {version = ">=4.13.2", markers = "python_version < \"3.11\"" [package.extras] docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] -test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8) ; platform_python_implementation == \"PyPy\" or platform_python_implementation == \"GraalVM\" or platform_python_implementation == \"CPython\" and sys_platform == \"win32\" and python_version >= \"3.13\"", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10) ; platform_python_implementation == \"CPython\""] [[package]] name = "yamlfix" @@ -1019,6 +1063,7 @@ version = "1.16.1" description = "A simple opionated yaml formatter that keeps your comments!" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "yamlfix-1.16.1-py3-none-any.whl", hash = "sha256:8c505ca27cf19181ca8943101b56b8e4ad58f47aa792fbab01339ededaddb7d2"}, {file = "yamlfix-1.16.1.tar.gz", hash = "sha256:f49ba70e457a1add6724a6859505d22f7f222f56f7e31f37822c530fc2e7ec94"}, @@ -1030,6 +1075,6 @@ maison = ">=1.4.0,<1.4.3" ruyaml = ">=0.91.0" [metadata] -lock-version = "2.0" +lock-version = "2.1" python-versions = "^3.9" -content-hash = "7ab03cbabe9011d7cbf14e3c328258a1977604214d85565a93c5008018cd199c" +content-hash = "350cf05513620283f7727ac4146751814bd61f42f08579c7048e17def8679a91" diff --git a/pyproject.toml b/pyproject.toml index 20a1a02..26083f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ python = "^3.9" click = "^8.1.8" hecdss = { version = ">=0.1.24", optional = true } # Via https://github.com/HydrologicEngineeringCenter/hec-python-library/blob/main/hec/shared.py#L9-10 cwms-python = { version = ">=0.8.0", optional = true} +pyyaml = "^6.0.2" [tool.poetry.group.dev.dependencies] black = "^24.2.0" From d0684f398a976e15ee29e4ede4a0138d3d4c546a Mon Sep 17 00:00:00 2001 From: Charles Graham SWT Date: Tue, 16 Sep 2025 16:45:09 -0500 Subject: [PATCH 02/25] Add initial templates --- .../templates/jinja/daily/_table.html.j2 | 64 +++++++++++++++++++ .../templates/jinja/daily/base.html.j2 | 33 ++++++++++ .../reporting/templates/jinja/report.html.j2 | 6 ++ .../templates/jinja/themes/_header.html.j2 | 19 ++++++ 4 files changed, 122 insertions(+) create mode 100644 cwmscli/reporting/templates/jinja/daily/_table.html.j2 create mode 100644 cwmscli/reporting/templates/jinja/daily/base.html.j2 create mode 100644 cwmscli/reporting/templates/jinja/report.html.j2 create mode 100644 cwmscli/reporting/templates/jinja/themes/_header.html.j2 diff --git a/cwmscli/reporting/templates/jinja/daily/_table.html.j2 b/cwmscli/reporting/templates/jinja/daily/_table.html.j2 new file mode 100644 index 0000000..581caaa --- /dev/null +++ b/cwmscli/reporting/templates/jinja/daily/_table.html.j2 @@ -0,0 +1,64 @@ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {% for basin_name, reservoirs in basins.items() %} + + + + {% for prefix, name in reservoirs %} + + + + + + + + + + + + + + + + + + + {% endfor %} + {% endfor %} + +
ReservoirPool Elevation (ft)Pool Limits (ft)Pool Occupied StorageInflow (cfs)Outflow (cfs)Pumpage (mgd)Evap (in)Precip (in)
8AM
FT-NGVD
24hr
Change
Top Of(%)Pool(ac-ft)Mean8AM
Release
Mean
Turbine
Daily
Gated
Average
Daily
Pump
24hr24hr
ConFlood
{{ basin_name }}
{{ name }}{{ values[prefix ~ '_elev'] }}{{ values[prefix ~ '_diff'] }}{{ ts(name="keys.elev.inst.1Hour.0.Ccp-Rev", office="SWT") }}{{ pool_limits.get(prefix, (None,None,None))[1] or '--' }}{{ pool_limits.get(prefix, (None,None,None))[2] or '--' }}{{ values[prefix ~ '_stor_pct'] }}{{ values[prefix ~ '_stor_letter'] }}{{ values[prefix ~ '_stor'] }}{{ values[prefix ~ '_inf'] }}{{ values[prefix ~ '_rel'] }}{{ values[prefix ~ '_turb'] }}{{ values[prefix ~ '_gat'] }}{{ values[prefix ~ '_pump'] }}{{ values[prefix ~ '_evap'] }}{{ values[prefix ~ '_rain'] }}
+
diff --git a/cwmscli/reporting/templates/jinja/daily/base.html.j2 b/cwmscli/reporting/templates/jinja/daily/base.html.j2 new file mode 100644 index 0000000..6b41bfd --- /dev/null +++ b/cwmscli/reporting/templates/jinja/daily/base.html.j2 @@ -0,0 +1,33 @@ + + + + + + + {% block title %}{{office}} — {{report.name}}{% endblock %} + + {% block head %}{% endblock %} + + +
+ {% block body %}{% endblock %} +
+ + + diff --git a/cwmscli/reporting/templates/jinja/report.html.j2 b/cwmscli/reporting/templates/jinja/report.html.j2 new file mode 100644 index 0000000..53ce4af --- /dev/null +++ b/cwmscli/reporting/templates/jinja/report.html.j2 @@ -0,0 +1,6 @@ +{% extends "daily/base.html.j2" %} +{% block title %}{{report.district}} {{report.name}}{% endblock %} +{% block body %} + {% include "themes/_header.html.j2" %} + {% include "daily/_table.html.j2" %} +{% endblock %} diff --git a/cwmscli/reporting/templates/jinja/themes/_header.html.j2 b/cwmscli/reporting/templates/jinja/themes/_header.html.j2 new file mode 100644 index 0000000..c9a35a7 --- /dev/null +++ b/cwmscli/reporting/templates/jinja/themes/_header.html.j2 @@ -0,0 +1,19 @@ +
+
+ + + + + + +
+ USACE + + {{ report.district }}
+ {{ report.name }}
+ Generated for {{ base_date.strftime("%d %b %Y @ %H%M") }} +
+ {{office}} +
+
+
From e66c7a75435fb675f15c6aec8bf234e144bc763a Mon Sep 17 00:00:00 2001 From: Charles Graham SWT Date: Tue, 16 Sep 2025 16:48:37 -0500 Subject: [PATCH 03/25] Update report yaml to extend to columns/rows --- report.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 report.yaml diff --git a/report.yaml b/report.yaml new file mode 100644 index 0000000..44255d5 --- /dev/null +++ b/report.yaml @@ -0,0 +1,29 @@ +# report.yaml +office: "SWT" # fallback: $OFFICE or $CWMS_OFFICE +cda_api_root: "https://cwms-data.usace.army.mil/cwms-data" # optional +begin: "24h" # ISO or relative like "24h", "3d", "90m" +# end: "2025-09-07:00:00-05:00" # optional; defaults to now (UTC) +default_unit: "EN" # can be overridden per column + +report: + district: "Tulsa District SWT" + name: "Daily Morning Reservoir Report" + logo_left: "https://www.swt-wc.usace.army.mil/images/logos/usace-logo-color.svg" + logo_right: "https://www.swt-wc.usace.army.mil/images/logos/tulsa.png" +projects: ["KEYS", "SKIA", "BROK"] + +columns: + - title: "Pool Elev (ft)" + tsid: "{project}.Elev.Inst.1Hour.0.Ccp" # template expands {project} + unit: "EN" + precision: 2 + key: "elev" + - title: "Tailwater (ft)" + tsid: "{project}.Tailwater.Inst.1Hour.0.Ccp" + precision: 2 + + - title: "Release (cfs)" + tsid: "{project}.Flow-Res Out.Ave.1Hour.1Hour.0.Ccp-Rev" + unit: "EN" + precision: 0 + key: "release" From d3ba838339a8d87291d2329712804726d76e156a Mon Sep 17 00:00:00 2001 From: Charles Graham SWT Date: Tue, 16 Sep 2025 16:50:32 -0500 Subject: [PATCH 04/25] Add initial reporting cmd/attempt render/scaffold design --- cwmscli/__main__.py | 2 + cwmscli/reporting/__init__.py | 280 ++++++++++++++++++++++++++ cwmscli/reporting/templates/README.md | 11 + 3 files changed, 293 insertions(+) create mode 100644 cwmscli/reporting/__init__.py create mode 100644 cwmscli/reporting/templates/README.md diff --git a/cwmscli/__main__.py b/cwmscli/__main__.py index e934503..09df75e 100644 --- a/cwmscli/__main__.py +++ b/cwmscli/__main__.py @@ -2,6 +2,7 @@ from cwmscli.commands import commands_cwms from cwmscli.getusgs import commands_getusgs +from cwmscli.reporting import reporting @click.group() @@ -16,3 +17,4 @@ def cli(): cli.add_command(commands_cwms.shefcritimport) cli.add_command(commands_cwms.csv2cwms_cmd) cli.add_command(commands_cwms.blob_group) +cli.add_command(reporting) diff --git a/cwmscli/reporting/__init__.py b/cwmscli/reporting/__init__.py new file mode 100644 index 0000000..c53dc55 --- /dev/null +++ b/cwmscli/reporting/__init__.py @@ -0,0 +1,280 @@ +import json +import os +from pathlib import Path + +import click + +from cwmscli.utils import api_key_option, api_root_option, get_api_key, office_option +from cwmscli.utils.deps import requires + + +# ---- Group ---- +@click.group("reporting", help="Create CWMS reports (Jinja, Repgen5, iText).") +@office_option +@api_root_option +@api_key_option +def reporting(office, api_root, api_key): + """ + Shared options for all reporting commands. Stores values in ctx.obj. + """ + print("Report!") + + +# ---- reporting scaffold ---- +@reporting.command( + "scaffold", help="Create a starter Jinja + Requests report in a folder." +) +@click.option( + "-d", + "--dir", + "out_dir", + default=".", + show_default=True, + help="Directory to create starter files.", +) +@requires( + { + "module": "jinja2", + "package": "Jinja2", + "version": "3.1.0", + "desc": "Templating engine", + "link": "https://palletsprojects.com/p/jinja/", + }, + { + "module": "requests", + "version": "2.31.0", + "desc": "HTTP client", + "link": "https://requests.readthedocs.io/", + }, +) +def scaffold(**kwargs): + from jinja2 import Template # lazy import so CLI stays snappy + + print(kwargs) + base = Path(kwargs.get("out_dir")) + base.mkdir(parents=True, exist_ok=True) + + # Minimal starter template (HTML) using variables and a small loop. + template = """ + + {{ title }} + +

{{ title }}

+

Office: {{ office }}

+

Generated at: {{ generated_at }}

+ +

Latest values

+
    + {% for item in series %} +
  • {{ item.name }}: {{ item.value }} {{ item.unit }} at {{ item.time }}
  • + {% endfor %} +
+ + +""" + (base / "report.html.j2").write_text(template, encoding="utf-8") + + # Starter config: which time series to fetch and how to label the report. + config = { + "title": "Sample CWMS Report", + "series": [ + { + "name": "KEYS.Elev.Inst.1Hour.0.Ccp-Rev", + "alias": "Keystone Elevation", + } + ], + # Optional begin/end ISO strings; if omitted, the renderer can choose defaults + "begin": None, + "end": None, + "unit_system": "EN", + } + (base / "report.json").write_text(json.dumps(config, indent=2), encoding="utf-8") + + # Tiny README + (base / "README.md").write_text( + """# Reporting Starter + +Files: +- `report.html.j2` — Jinja2 template +- `report.json` — configuration (title, series list, optional begin/end) +- Use: `cwms-cli reporting render -d reporting_starter -o out.html` + +""", + encoding="utf-8", + ) + + click.echo(f"Scaffold created in: {base.resolve()}") + + +# ---- reporting render ---- +@reporting.command("render", help="Render a Jinja template using CWMS Data API.") +@click.option( + "-d", + "--dir", + "in_dir", + default=".", + show_default=True, + help="Directory with report.html.j2 and report.json.", +) +@click.option( + "-t", + "--template", + default="report.html.j2", + show_default=True, + help="Template filename inside the directory.", +) +@click.option( + "-c", + "--config", + default="report.json", + show_default=True, + help="Config JSON filename inside the directory.", +) +@click.option( + "--output", + "out_file", + default="report.html", + show_default=True, + help="Output file.", +) +@click.option("--begin", help="Override begin ISO8601 (optional).") +@click.option("--end", help="Override end ISO8601 (optional).") +@office_option +@api_root_option +@requires( + { + "module": "jinja2", + "package": "Jinja2", + "version": "3.1.0", + "desc": "Templating engine", + }, + { + "module": "requests", + "version": "2.31.0", + "desc": "HTTP client", + }, +) +def render_report(**kwargs): + """ + Loads config, calls CWMS Data API for each series, renders template. + """ + from datetime import datetime, timezone + + import requests + from jinja2 import Environment, FileSystemLoader, select_autoescape + + office = kwargs.get("office") + api_root = kwargs.get("api_root").rstrip("/") + api_key = kwargs.get("api_key") + + base = Path(kwargs.get("in_dir")) + cfg = json.loads((base / kwargs.get("config")).read_text(encoding="utf-8")) + + # Allow CLI begin/end overrides + if kwargs.get("begin"): + cfg["begin"] = kwargs.get("begin") + if kwargs.get("end"): + cfg["end"] = kwargs.get("end") + + headers = {} + # if api_key: + # headers["Authorization"] = f"Bearer {api_key}" + + # Minimal fetcher: latest value or bounded time-window (kept simple here) + def fetch_latest(name: str): + # If you want last value, ask for a tiny window ending "now" + # You can refine with timeseries/values GET params as needed + params = { + "name": name, + "office": office, + } + if cfg.get("begin"): + params["begin"] = kwargs.get("begin") + if cfg.get("end"): + params["end"] = kwargs.get("end") + if cfg.get("unit_system"): + params["unit-system"] = cfg.get("unit_system") + + url = f"{api_root}/timeseries" + r = requests.get(url, params=params, headers=headers, timeout=30) + r.raise_for_status() + data = r.json() + + # Normalize a simple return shape (you’ll adapt to your exact API) + # Expecting something like: { "values": [ [epoch_ms, value, qual], ... ], "units": "ft", ... } + values = data.get("values") or data.get("value-pairs") or [] + unit = data.get("units") or data.get("unit") or "" + if values: + # Try to handle both [epoch,value,...] or object shapes + last = values[-1] + if isinstance(last, (list, tuple)) and len(last) >= 2: + epoch_ms, val = last[0], last[1] + t = datetime.fromtimestamp( + epoch_ms / 1000.0, tz=timezone.utc + ).isoformat() + return val, t, unit + elif isinstance(last, dict): + val = last.get("value") + t = last.get("time") or last.get("date-time") or "" + return val, t, unit + return None, None, unit + + series_results = [] + for s in cfg.get("series", []): + val, t, unit = fetch_latest(s["name"]) + series_results.append( + { + "name": s.get("alias") or s["name"], + "value": val, + "time": t, + "unit": unit, + } + ) + + env = Environment( + loader=FileSystemLoader(str(base)), + autoescape=select_autoescape(["html", "xml"]), + ) + tmpl = env.get_template(kwargs.get("template")) + + html = tmpl.render( + title=cfg.get("title", "CWMS Report"), + office=office, + generated_at=datetime.utcnow().isoformat() + "Z", + series=series_results, + cfg=cfg, + ) + Path(kwargs.get("out_file")).write_text(html, encoding="utf-8") + click.echo(f"Wrote {Path(kwargs.get('out_file')).resolve()}") + + +# ---- reporting repgen ---- +@reporting.command("repgen", help="Create a report using Repgen5 (stub).") +@requires( + { + "module": "jinja2", + "package": "Jinja2", + "version": "3.1.0", + "desc": "Templating for pre/post-processing", + }, + # Add your repgen5 dependency discovery here when ready (e.g., Java/JAR presence) +) +@click.option("--template", help="Repgen template/file to execute (future).") +def generate_repgen(template): + click.echo("Repgen5 integration coming soon.") + + +# ---- reporting itext ---- +@reporting.command("itext", help="Create a report using iText (stub).") +@requires( + # Example: you might end up calling into Java; jpype could be one approach + { + "module": "jpype", + "version": "1.5.0", + "desc": "Bridge to call Java iText from Python (optional approach).", + "link": "https://jpype.readthedocs.io/", + }, +) +@click.option("--template", help="iText template/file to execute (future).") +def generate_itext(template): + click.echo("iText integration coming soon.") diff --git a/cwmscli/reporting/templates/README.md b/cwmscli/reporting/templates/README.md new file mode 100644 index 0000000..745d338 --- /dev/null +++ b/cwmscli/reporting/templates/README.md @@ -0,0 +1,11 @@ +# Jinja Templates + +To render and view the templates with proper syntax highlighting for various languages install: + +https://marketplace.visualstudio.com/items?itemName=samuelcolvin.jinjahtml + +This will map Jinja template files to the appropriate language mode in Visual Studio Code, enabling syntax highlighting and other language features. + +## HINTS + +- If you have the DJANGO extension installed it will conflict with the jinja extension listed above \ No newline at end of file From 661142e95da9056e0aeadd6ebcdb7762108dbc45 Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Tue, 16 Sep 2025 23:40:42 -0500 Subject: [PATCH 05/25] Change to a default template given a config, update internal daily template to accept dynamic input --- cwmscli/__main__.py | 4 +- cwmscli/reporting/__init__.py | 678 ++++++++++++------ cwmscli/reporting/configs/daily.yaml | 41 ++ .../templates/jinja/daily/_table.html.j2 | 71 +- .../templates/jinja/daily/base.html.j2 | 1 - 5 files changed, 500 insertions(+), 295 deletions(-) create mode 100644 cwmscli/reporting/configs/daily.yaml diff --git a/cwmscli/__main__.py b/cwmscli/__main__.py index 09df75e..c13df87 100644 --- a/cwmscli/__main__.py +++ b/cwmscli/__main__.py @@ -2,7 +2,7 @@ from cwmscli.commands import commands_cwms from cwmscli.getusgs import commands_getusgs -from cwmscli.reporting import reporting +from cwmscli.reporting import reporting_cli @click.group() @@ -17,4 +17,4 @@ def cli(): cli.add_command(commands_cwms.shefcritimport) cli.add_command(commands_cwms.csv2cwms_cmd) cli.add_command(commands_cwms.blob_group) -cli.add_command(reporting) +cli.add_command(reporting_cli) diff --git a/cwmscli/reporting/__init__.py b/cwmscli/reporting/__init__.py index c53dc55..630ed5a 100644 --- a/cwmscli/reporting/__init__.py +++ b/cwmscli/reporting/__init__.py @@ -1,280 +1,486 @@ -import json +from __future__ import annotations + +import math import os -from pathlib import Path +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Iterable, List, Optional import click +import cwms +import pandas as pd +import yaml -from cwmscli.utils import api_key_option, api_root_option, get_api_key, office_option from cwmscli.utils.deps import requires -# ---- Group ---- -@click.group("reporting", help="Create CWMS reports (Jinja, Repgen5, iText).") -@office_option -@api_root_option -@api_key_option -def reporting(office, api_root, api_key): +# ---------- Models ---------- +@dataclass +class ProjectSpec: + location_id: str + href: Optional[str] = None + office: Optional[str] = None + + +@dataclass +class ColumnSpec: + title: str + tsid: str + unit: Optional[str] = None + precision: Optional[int] = None + key: str = field(default="") + office: Optional[str] = None + location_id: Optional[str] = None + + +@dataclass +class ReportSpec: + district: str + name: str + logo_left: Optional[str] = None + logo_right: Optional[str] = None + + +@dataclass +class Config: + office: str + cda_api_root: Optional[str] = None + report: ReportSpec | Dict[str, Any] | None = None + projects: List[ProjectSpec] = field(default_factory=list) + columns: List[ColumnSpec] = field(default_factory=list) + begin: Optional[str] = None # ISO or relative like "24h" + end: Optional[str] = None # ISO + default_unit: str = "EN" + + @staticmethod + def from_yaml(path: str) -> "Config": + with open(path, "r", encoding="utf-8") as f: + raw = yaml.safe_load(f) or {} + # env fallbacks + office = ( + raw.get("office") + or os.getenv("OFFICE") + or os.getenv("CWMS_OFFICE") + or "SWT" + ) + cda_api_root = raw.get("cda_api_root") or os.getenv("CDA_API_ROOT") + report_block = raw.get("report") or {} + # normalize report to ReportSpec + report = ReportSpec( + district=report_block.get("district", office), + name=report_block.get("name", "Daily Report"), + logo_left=report_block.get("logo_left"), + logo_right=report_block.get("logo_right"), + ) + # columns + cols = [] + for i, c in enumerate(raw.get("columns", [])): + cols.append( + ColumnSpec( + title=c.get("title") or c.get("name") or f"Col{i+1}", + tsid=c["tsid"], + unit=c.get("unit"), + precision=c.get("precision"), + key=c.get("key") or c.get("title") or f"c{i+1}", + office=c.get("office"), + location_id=c.get("location_id"), + ) + ) + projects_raw = raw.get("projects", []) + projects: List[ProjectSpec] = [] + for p in projects_raw: + if isinstance(p, str): + projects.append(ProjectSpec(location_id=p)) + elif isinstance(p, dict): + projects.append( + ProjectSpec( + location_id=p.get("location_id") + or p.get("name") + or p.get("id"), + href=p.get("href"), + office=p.get("office"), + ) + ) + else: + raise click.BadParameter(f"Invalid project entry: {p!r}") + + return Config( + office=office, + cda_api_root=cda_api_root, + report=report, + projects=projects, + columns=cols, + begin=raw.get("begin"), + end=raw.get("end"), + default_unit=raw.get("default_unit") or "EN", + ) + + +# ---------- Helpers ---------- +def _parse_time_or_relative(s: Optional[str]) -> Optional[datetime]: """ - Shared options for all reporting commands. Stores values in ctx.obj. + Accepts ISO strings (with or without tz) or relative like "24h", "3d", "90m". + Returns timezone-aware UTC datetimes (or None). """ - print("Report!") + if not s: + return None + s = str(s).strip() + # relative + if s.endswith(("h", "m", "d")) and s[:-1].isdigit(): + amount = int(s[:-1]) + unit = s[-1] + now = datetime.now(timezone.utc) + if unit == "h": + return now - timedelta(hours=amount) + if unit == "m": + return now - timedelta(minutes=amount) + if unit == "d": + return now - timedelta(days=amount) + # ISO parse + try: + dt = datetime.fromisoformat(s) + except ValueError: + raise click.BadParameter(f"Invalid datetime: {s}") + # ensure tz-aware -> UTC + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) -# ---- reporting scaffold ---- -@reporting.command( - "scaffold", help="Create a starter Jinja + Requests report in a folder." -) -@click.option( - "-d", - "--dir", - "out_dir", - default=".", - show_default=True, - help="Directory to create starter files.", -) -@requires( - { - "module": "jinja2", - "package": "Jinja2", - "version": "3.1.0", - "desc": "Templating engine", - "link": "https://palletsprojects.com/p/jinja/", - }, - { - "module": "requests", - "version": "2.31.0", - "desc": "HTTP client", - "link": "https://requests.readthedocs.io/", - }, -) -def scaffold(**kwargs): - from jinja2 import Template # lazy import so CLI stays snappy - - print(kwargs) - base = Path(kwargs.get("out_dir")) - base.mkdir(parents=True, exist_ok=True) - - # Minimal starter template (HTML) using variables and a small loop. - template = """ - - {{ title }} - -

{{ title }}

-

Office: {{ office }}

-

Generated at: {{ generated_at }}

- -

Latest values

-
    - {% for item in series %} -
  • {{ item.name }}: {{ item.value }} {{ item.unit }} at {{ item.time }}
  • - {% endfor %} -
- - -""" - (base / "report.html.j2").write_text(template, encoding="utf-8") - - # Starter config: which time series to fetch and how to label the report. - config = { - "title": "Sample CWMS Report", - "series": [ +def _ensure_tz(dt: Optional[datetime]) -> Optional[datetime]: + if dt is None: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + + +def _format_number(x: Any, precision: Optional[int]) -> str: + if x is None or (isinstance(x, float) and (math.isnan(x) or math.isinf(x))): + return "-" + try: + if precision is None: + return f"{x}" + fmt = f"{{:.{precision}f}}" + return fmt.format(float(x)) + except Exception: + return f"{x}" + + +def _expand_tsid(tsid_template: str, project: str) -> str: + """ + If tsid_template contains '{project}', substitute it. + Otherwise return as-is (full TSIDs remain unchanged). + """ + if "{project}" in tsid_template: + return tsid_template.format(project=project) + return tsid_template + + +def _fetch_multi_df( + tsids: List[str], + office: str, + unit: str, + begin: Optional[datetime], + end: Optional[datetime], +) -> pd.DataFrame: + """ + Wrapper around cwms.get_multi_timeseries_df, always returns a melted frame: + columns: ['date-time','name','value','quality-code'] (depending on cwms-python version) + """ + df = cwms.get_multi_timeseries_df( + ts_ids=tsids, + office_id=office, + unit=unit, + begin=begin, + end=end, + melted=True, + ) + # expected columns: 'date-time', 'name', 'value' (possibly 'quality-code') + # make sure types are reasonable + if "date-time" in df.columns: + df["date-time"] = pd.to_datetime(df["date-time"], utc=True, errors="coerce") + return df + + +def _last_values_by_name(df: pd.DataFrame) -> Dict[str, float | None]: + """ + For a melted dataframe with 'name' and 'date-time' and 'value', + return the last (by time) non-null value for each name. + """ + if df.empty: + return {} + work = df.dropna(subset=["value"]) + if work.empty: + return {} + # sort then groupby tail(1) + print(work.columns, flush=True) + work = work.sort_values(["ts_id", "date-time"]) + last = work.groupby("ts_id").tail(1) + return dict(zip(last["ts_id"], last["value"])) + + +def _render_template( + template_dir: Optional[str], + template_name: str | None, + context: Dict[str, Any], +) -> str: + """ + Try user-specified template directory first; if not provided or missing, + fall back to package templates (if you ship any). For now, we only support + user-supplied templates or very simple built-in fallback. + """ + import jinja2 + + loaders: List[jinja2.BaseLoader] = [] + # Use the built-in package templates if no user dir is given + if not template_dir: + pkg_dir = os.path.abspath( + os.path.join(os.path.dirname(__file__), "templates", "jinja") + ) + print(pkg_dir, flush=True) + if os.path.isdir(pkg_dir): + loaders.append(jinja2.FileSystemLoader(pkg_dir)) + # User-specified template dir (highest priority) + if template_dir and os.path.isdir(template_dir): + loaders.append(jinja2.FileSystemLoader(template_dir)) + + # Fallback minimal inline template if not found + env = jinja2.Environment( + loader=jinja2.ChoiceLoader(loaders) if loaders else None, + autoescape=True, + trim_blocks=True, + lstrip_blocks=True, + ) + + try: + if not template_name: + template_name = "report.html.j2" + tmpl = env.get_template(template_name) + return tmpl.render(**context) + except Exception as e: + # ultra-simple built-in fallback table + # (lets the command succeed even if no templates are set up yet) + cols = context["columns"] + rows = context["rows"] + data = context["data"] + title = f'{context.get("report",{}).get("district","") or context.get("office","") } {context.get("report",{}).get("name","Report")}' + head = ( + "Project" + + "".join(f"{c['title']}" for c in cols) + + "" + ) + body = [] + for proj in rows: + tds = [f"{proj}"] + for c in cols: + tds.append(f"{data.get(proj,{}).get(c['key'],'-')}") + body.append("" + "".join(tds) + "") + html = f""" +{title} + + +

{title}

+{head}{''.join(body)}
+""" + click.echo( + f"[reporting] Using built-in fallback template because '{template_name}' was not found.\nError: ({e})", + err=True, + ) + return html + + +# ---------- Main routine ---------- +def build_report_table( + config: Config, begin: Optional[datetime], end: Optional[datetime] +) -> Dict[str, Any]: + # rows remain as simple strings for template compatibility + rows: List[str] = [p.location_id for p in config.projects] + if not rows: + raise click.UsageError("No 'projects' configured in YAML.") + + # quick lookup + proj_by_id: Dict[str, ProjectSpec] = {p.location_id: p for p in config.projects} + + # columns with fallback office/unit + col_defs: List[Dict[str, Any]] = [] + for c in config.columns: + col_defs.append( { - "name": "KEYS.Elev.Inst.1Hour.0.Ccp-Rev", - "alias": "Keystone Elevation", + "title": c.title, + "key": c.key or c.title, + "precision": c.precision, + "unit": c.unit or config.default_unit, + "tsid_template": c.tsid, + "office": c.office or config.office, # <- fallback here } - ], - # Optional begin/end ISO strings; if omitted, the renderer can choose defaults - "begin": None, - "end": None, - "unit_system": "EN", - } - (base / "report.json").write_text(json.dumps(config, indent=2), encoding="utf-8") + ) - # Tiny README - (base / "README.md").write_text( - """# Reporting Starter + # group tsids by (office, unit) + group_tsids: Dict[tuple, List[str]] = {} # (office, unit) -> tsids + backref: Dict[tuple, List[tuple]] = ( + {} + ) # (office, unit, tsid) -> [(project_id, column_key)] -Files: -- `report.html.j2` — Jinja2 template -- `report.json` — configuration (title, series list, optional begin/end) -- Use: `cwms-cli reporting render -d reporting_starter -o out.html` + for proj_id in rows: + for c in col_defs: + tsid = _expand_tsid(c["tsid_template"], proj_id) + k = (c["office"], c["unit"]) + group_tsids.setdefault(k, []) + if tsid not in group_tsids[k]: + group_tsids[k].append(tsid) + backref.setdefault((c["office"], c["unit"], tsid), []).append( + (proj_id, c["key"]) + ) -""", - encoding="utf-8", - ) + # fetch & last values + last_value_by_key: Dict[tuple, float | None] = {} + for (office, unit), tsids in group_tsids.items(): + if not tsids: + continue + df = _fetch_multi_df(tsids, office, unit, begin, end) + last_vals = _last_values_by_name(df) + for ts in tsids: + last_value_by_key[(office, unit, ts)] = last_vals.get(ts) + + # prefetch project locations once and graft href + proj_locations: Dict[str, Dict[str, Any]] = {} + for proj_id in rows: + proj = proj_by_id[proj_id] + proj_office = proj.office or config.office + try: + loc = cwms.get_location(office_id=proj_office, location_id=proj_id) + loc_json = ( + getattr(loc, "json", None) or loc + ) # cwms-python returns object w/ .json + if isinstance(loc_json, dict): + loc_json = {**loc_json} + if proj.href: + loc_json["href"] = proj.href + else: + loc_json = {"name": proj_id, "href": proj.href} + except Exception: + loc_json = {"name": proj_id, "href": proj.href} + proj_locations[proj_id] = loc_json - click.echo(f"Scaffold created in: {base.resolve()}") + # build the table payload + table: Dict[str, Dict[str, Any]] = {proj_id: {} for proj_id in rows} + for (office, unit, tsid), pairs in backref.items(): + raw_val = last_value_by_key.get((office, unit, tsid)) + for proj_id, col_key in pairs: + col = next((c for c in col_defs if c["key"] == col_key), None) + precision = col.get("precision") if col else None + table[proj_id][col_key] = _format_number(raw_val, precision) + + # attach location block (with href) per project row + for proj_id in rows: + table[proj_id]["location"] = proj_locations.get(proj_id, {"name": proj_id}) + + return { + "columns": col_defs, + "rows": rows, # still a list of project IDs for your template loop + "data": table, # data[proj]["location"]["href"] now exists (if provided) + } -# ---- reporting render ---- -@reporting.command("render", help="Render a Jinja template using CWMS Data API.") + +# ---------- Click entry ---------- +@click.command( + name="reporting", + help="Render a CWMS timeseries report to HTML using a YAML config and Jinja2.", +) @click.option( - "-d", - "--dir", - "in_dir", - default=".", - show_default=True, - help="Directory with report.html.j2 and report.json.", + "--config", + "-c", + "config_path", + required=True, + type=click.Path(exists=True, dir_okay=False), + help="Path to report YAML.", ) @click.option( + "--template-dir", "-t", + "template_dir", + type=click.Path(exists=True, file_okay=False), + help="Directory containing Jinja templates (e.g., templates/jinja).", +) +@click.option( "--template", - default="report.html.j2", - show_default=True, - help="Template filename inside the directory.", + "-n", + "template_name", + default=None, + help="Template filename to render (relative to --template-dir). Default: report.html.j2", ) @click.option( - "-c", - "--config", - default="report.json", - show_default=True, - help="Config JSON filename inside the directory.", + "--begin", + help='Override begin time (ISO or relative like "24h"). If omitted, uses YAML or defaults.', ) +@click.option("--end", help="Override end time (ISO). If omitted, uses YAML or now.") @click.option( - "--output", - "out_file", + "--out", + "-o", + "out_path", default="report.html", show_default=True, - help="Output file.", + type=click.Path(dir_okay=False), + help="Output HTML path.", ) -@click.option("--begin", help="Override begin ISO8601 (optional).") -@click.option("--end", help="Override end ISO8601 (optional).") -@office_option -@api_root_option @requires( { "module": "jinja2", "package": "Jinja2", "version": "3.1.0", - "desc": "Templating engine", - }, - { - "module": "requests", - "version": "2.31.0", - "desc": "HTTP client", + "desc": "Templating for pre/post-processing", }, ) -def render_report(**kwargs): - """ - Loads config, calls CWMS Data API for each series, renders template. - """ - from datetime import datetime, timezone - - import requests - from jinja2 import Environment, FileSystemLoader, select_autoescape - - office = kwargs.get("office") - api_root = kwargs.get("api_root").rstrip("/") - api_key = kwargs.get("api_key") - - base = Path(kwargs.get("in_dir")) - cfg = json.loads((base / kwargs.get("config")).read_text(encoding="utf-8")) - - # Allow CLI begin/end overrides - if kwargs.get("begin"): - cfg["begin"] = kwargs.get("begin") - if kwargs.get("end"): - cfg["end"] = kwargs.get("end") - - headers = {} - # if api_key: - # headers["Authorization"] = f"Bearer {api_key}" - - # Minimal fetcher: latest value or bounded time-window (kept simple here) - def fetch_latest(name: str): - # If you want last value, ask for a tiny window ending "now" - # You can refine with timeseries/values GET params as needed - params = { - "name": name, - "office": office, - } - if cfg.get("begin"): - params["begin"] = kwargs.get("begin") - if cfg.get("end"): - params["end"] = kwargs.get("end") - if cfg.get("unit_system"): - params["unit-system"] = cfg.get("unit_system") - - url = f"{api_root}/timeseries" - r = requests.get(url, params=params, headers=headers, timeout=30) - r.raise_for_status() - data = r.json() - - # Normalize a simple return shape (you’ll adapt to your exact API) - # Expecting something like: { "values": [ [epoch_ms, value, qual], ... ], "units": "ft", ... } - values = data.get("values") or data.get("value-pairs") or [] - unit = data.get("units") or data.get("unit") or "" - if values: - # Try to handle both [epoch,value,...] or object shapes - last = values[-1] - if isinstance(last, (list, tuple)) and len(last) >= 2: - epoch_ms, val = last[0], last[1] - t = datetime.fromtimestamp( - epoch_ms / 1000.0, tz=timezone.utc - ).isoformat() - return val, t, unit - elif isinstance(last, dict): - val = last.get("value") - t = last.get("time") or last.get("date-time") or "" - return val, t, unit - return None, None, unit - - series_results = [] - for s in cfg.get("series", []): - val, t, unit = fetch_latest(s["name"]) - series_results.append( - { - "name": s.get("alias") or s["name"], - "value": val, - "time": t, - "unit": unit, - } - ) +def reporting_cli(config_path, template_dir, template_name, begin, end, out_path): + # Load config + cfg = Config.from_yaml(config_path) - env = Environment( - loader=FileSystemLoader(str(base)), - autoescape=select_autoescape(["html", "xml"]), - ) - tmpl = env.get_template(kwargs.get("template")) - - html = tmpl.render( - title=cfg.get("title", "CWMS Report"), - office=office, - generated_at=datetime.utcnow().isoformat() + "Z", - series=series_results, - cfg=cfg, + # Resolve time window + cfg_begin = ( + _parse_time_or_relative(begin) if begin else _parse_time_or_relative(cfg.begin) ) - Path(kwargs.get("out_file")).write_text(html, encoding="utf-8") - click.echo(f"Wrote {Path(kwargs.get('out_file')).resolve()}") + cfg_end = _parse_time_or_relative(end) if end else _parse_time_or_relative(cfg.end) + if cfg_end is None: + cfg_end = datetime.now(timezone.utc) + if cfg_begin is None: + # CWMS default is end-24h, but we make it explicit here + cfg_begin = cfg_end - timedelta(hours=24) + # Configure cwms client API root if provided + cwms.init_session(api_root=cfg.cda_api_root) -# ---- reporting repgen ---- -@reporting.command("repgen", help="Create a report using Repgen5 (stub).") -@requires( - { - "module": "jinja2", - "package": "Jinja2", - "version": "3.1.0", - "desc": "Templating for pre/post-processing", - }, - # Add your repgen5 dependency discovery here when ready (e.g., Java/JAR presence) -) -@click.option("--template", help="Repgen template/file to execute (future).") -def generate_repgen(template): - click.echo("Repgen5 integration coming soon.") + # Build table data + print(cfg) + table_ctx = build_report_table(cfg, cfg_begin, cfg_end) + # Render + base_date = cfg_end.astimezone(timezone.utc) + context = { + "office": cfg.office, + "report": dataclasses_asdict(cfg.report), + "base_date": base_date, + **table_ctx, + } + print("TEMPLATE DIR", template_dir) + html = _render_template(template_dir, template_name, context) -# ---- reporting itext ---- -@reporting.command("itext", help="Create a report using iText (stub).") -@requires( - # Example: you might end up calling into Java; jpype could be one approach - { - "module": "jpype", - "version": "1.5.0", - "desc": "Bridge to call Java iText from Python (optional approach).", - "link": "https://jpype.readthedocs.io/", - }, -) -@click.option("--template", help="iText template/file to execute (future).") -def generate_itext(template): - click.echo("iText integration coming soon.") + with open(out_path, "w", encoding="utf-8") as f: + f.write(html) + click.echo(f"Wrote {out_path}") + + +def dataclasses_asdict(obj): + if obj is None: + return None + if hasattr(obj, "__dataclass_fields__"): + return { + fld: dataclasses_asdict(getattr(obj, fld)) + for fld in obj.__dataclass_fields__ + } + if isinstance(obj, (list, tuple)): + return [dataclasses_asdict(x) for x in obj] + if isinstance(obj, dict): + return {k: dataclasses_asdict(v) for k, v in obj.items()} + return obj diff --git a/cwmscli/reporting/configs/daily.yaml b/cwmscli/reporting/configs/daily.yaml new file mode 100644 index 0000000..e07f24c --- /dev/null +++ b/cwmscli/reporting/configs/daily.yaml @@ -0,0 +1,41 @@ +# report.yaml +office: "SWT" # fallback: $OFFICE or $CWMS_OFFICE +cda_api_root: "https://cwms-data.usace.army.mil/cwms-data/" # optional +begin: "24h" # ISO or relative like "24h", "3d", "90m" +# end: "2025-09-16T11:00:00-05:00" # optional; defaults to now (UTC) + +default_unit: "EN" # can be overridden per column + +report: + district: "Tulsa District SWT" + name: "Daily Morning Reservoir Report" + logo_left: "https://www.swt-wc.usace.army.mil/images/logos/usace-logo-color.svg" + logo_right: "https://www.swt-wc.usace.army.mil/images/logos/tulsa.png" + +projects: + - location_id: "KEYS" + href: "https://www.swt-wc.usace.army.mil/keys.lakepage.html" + - location_id: "SKIA" + href: "https://www.swt-wc.usace.army.mil/skia.lakepage.html" + - location_id: "BROK" + href: "https://www.swt-wc.usace.army.mil/brok.lakepage.html" + +columns: + - title: "Pool Elev (ft)" + location_id: "KEYS" + tsid: "{project}.Elev.Inst.1Hour.0.Ccp-Rev" + office: "SWT" + unit: "EN" + precision: 2 + key: "elev" + + # - title: "Release (cfs)" + # tsid: "{project}.Flow-Res Out.Ave.1Hour.1Hour.0.Ccp-Rev" + # unit: "EN" + # precision: 0 + # key: "release" + + # - title: "Tailwater (ft)" + # tsid: "{project}.Tailwater.Inst.1Hour.0.Ccp-Rev" + # precision: 2 + # key: "tailwater" diff --git a/cwmscli/reporting/templates/jinja/daily/_table.html.j2 b/cwmscli/reporting/templates/jinja/daily/_table.html.j2 index 581caaa..b0f8eba 100644 --- a/cwmscli/reporting/templates/jinja/daily/_table.html.j2 +++ b/cwmscli/reporting/templates/jinja/daily/_table.html.j2 @@ -1,64 +1,23 @@ -
- +
+
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + {% for col in columns %} + + {% endfor %} - {% for basin_name, reservoirs in basins.items() %} - - + {% for proj in rows %} + + + {% for col in columns %} + + {% endfor %} - {% for prefix, name in reservoirs %} - - - - - - - - - - - - - - - - - - - {% endfor %} {% endfor %}
ReservoirPool Elevation (ft)Pool Limits (ft)Pool Occupied StorageInflow (cfs)Outflow (cfs)Pumpage (mgd)Evap (in)Precip (in)
8AM
FT-NGVD
24hr
Change
Top Of(%)Pool(ac-ft)Mean8AM
Release
Mean
Turbine
Daily
Gated
Average
Daily
Pump
24hr24hr
ConFlood
Project{{ col.title }}
{{ basin_name }}
+ {{data[proj]["location"]["public-name"]}}{{ data[proj][col.key] }}
{{ name }}{{ values[prefix ~ '_elev'] }}{{ values[prefix ~ '_diff'] }}{{ ts(name="keys.elev.inst.1Hour.0.Ccp-Rev", office="SWT") }}{{ pool_limits.get(prefix, (None,None,None))[1] or '--' }}{{ pool_limits.get(prefix, (None,None,None))[2] or '--' }}{{ values[prefix ~ '_stor_pct'] }}{{ values[prefix ~ '_stor_letter'] }}{{ values[prefix ~ '_stor'] }}{{ values[prefix ~ '_inf'] }}{{ values[prefix ~ '_rel'] }}{{ values[prefix ~ '_turb'] }}{{ values[prefix ~ '_gat'] }}{{ values[prefix ~ '_pump'] }}{{ values[prefix ~ '_evap'] }}{{ values[prefix ~ '_rain'] }}
-
+ \ No newline at end of file diff --git a/cwmscli/reporting/templates/jinja/daily/base.html.j2 b/cwmscli/reporting/templates/jinja/daily/base.html.j2 index 6b41bfd..2ee7221 100644 --- a/cwmscli/reporting/templates/jinja/daily/base.html.j2 +++ b/cwmscli/reporting/templates/jinja/daily/base.html.j2 @@ -1,4 +1,3 @@ - From 3b0b54aa252bba38e850cdd283e3daf7cce2b28a Mon Sep 17 00:00:00 2001 From: Charles Graham SWT Date: Wed, 17 Sep 2025 10:30:42 -0500 Subject: [PATCH 06/25] Update to allow for levels and per column href with specific begin/end --- cwmscli/reporting/__init__.py | 481 ++++++++++++++++++++------- cwmscli/reporting/configs/daily.yaml | 51 +-- report.yaml | 29 -- 3 files changed, 384 insertions(+), 177 deletions(-) delete mode 100644 report.yaml diff --git a/cwmscli/reporting/__init__.py b/cwmscli/reporting/__init__.py index 630ed5a..5c6a424 100644 --- a/cwmscli/reporting/__init__.py +++ b/cwmscli/reporting/__init__.py @@ -2,9 +2,11 @@ import math import os +import traceback from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone -from typing import Any, Dict, Iterable, List, Optional +from typing import Any, Dict, List, Optional +from zoneinfo import ZoneInfo import click import cwms @@ -14,7 +16,6 @@ from cwmscli.utils.deps import requires -# ---------- Models ---------- @dataclass class ProjectSpec: location_id: str @@ -22,23 +23,28 @@ class ProjectSpec: office: Optional[str] = None +@dataclass +class ReportSpec: + district: str + name: str + logo_left: Optional[str] = None + logo_right: Optional[str] = None + + @dataclass class ColumnSpec: title: str - tsid: str + key: str + tsid: Optional[str] = None + level: Optional[str] = None unit: Optional[str] = None precision: Optional[int] = None - key: str = field(default="") office: Optional[str] = None location_id: Optional[str] = None - - -@dataclass -class ReportSpec: - district: str - name: str - logo_left: Optional[str] = None - logo_right: Optional[str] = None + href: Optional[str] = None + missing: Optional[str] = None + undefined: Optional[str] = None + target_time: Optional[str] = None @dataclass @@ -48,44 +54,57 @@ class Config: report: ReportSpec | Dict[str, Any] | None = None projects: List[ProjectSpec] = field(default_factory=list) columns: List[ColumnSpec] = field(default_factory=list) - begin: Optional[str] = None # ISO or relative like "24h" - end: Optional[str] = None # ISO + + begin: Optional[str] = None + end: Optional[str] = None + + target_time: Optional[str] = None + time_epsilon_minutes: int = 5 + default_unit: str = "EN" + missing: str = "----" + undefined: str = "--NA--" + time_zone: Optional[str] = None @staticmethod def from_yaml(path: str) -> "Config": with open(path, "r", encoding="utf-8") as f: raw = yaml.safe_load(f) or {} - # env fallbacks + office = ( raw.get("office") or os.getenv("OFFICE") or os.getenv("CWMS_OFFICE") or "SWT" ) - cda_api_root = raw.get("cda_api_root") or os.getenv("CDA_API_ROOT") + report_block = raw.get("report") or {} - # normalize report to ReportSpec report = ReportSpec( district=report_block.get("district", office), name=report_block.get("name", "Daily Report"), logo_left=report_block.get("logo_left"), logo_right=report_block.get("logo_right"), ) - # columns - cols = [] + + cols: List[ColumnSpec] = [] for i, c in enumerate(raw.get("columns", [])): cols.append( ColumnSpec( title=c.get("title") or c.get("name") or f"Col{i+1}", - tsid=c["tsid"], + key=c.get("key") or c.get("title") or f"c{i+1}", + tsid=c.get("tsid"), + level=c.get("level"), unit=c.get("unit"), precision=c.get("precision"), - key=c.get("key") or c.get("title") or f"c{i+1}", office=c.get("office"), location_id=c.get("location_id"), + href=c.get("href"), + missing=c.get("missing"), + undefined=c.get("undefined"), + target_time=c.get("target_time"), ) ) + projects_raw = raw.get("projects", []) projects: List[ProjectSpec] = [] for p in projects_raw: @@ -106,17 +125,112 @@ def from_yaml(path: str) -> "Config": return Config( office=office, - cda_api_root=cda_api_root, + cda_api_root=raw.get("cda_api_root") or os.getenv("CDA_API_ROOT"), report=report, projects=projects, columns=cols, begin=raw.get("begin"), end=raw.get("end"), + target_time=raw.get("target_time"), + time_epsilon_minutes=int(raw.get("time_epsilon_minutes") or 5), default_unit=raw.get("default_unit") or "EN", + missing=raw.get("missing") or "----", + undefined=raw.get("undefined") or "--NA--", + time_zone=raw.get("time_zone"), + ) + + +def _parse_target_like( + s: Optional[str], default_tz: Optional[str] +) -> Optional[datetime]: + """ + Accepts: + - ISO (with/without tz): '2025-09-17T08:00:00-05:00', '2025-09-17T13:00Z' + - 'HHMM YYYY-MM-DD [TZ]', 'HHMM MM/DD/YYYY [TZ]', 'HH:MM MM/DD/YYYY [TZ]' + - '0800 09/17/2025 America/Chicago' + - 'today 08:00 [TZ]' or 'yesterday 08:00 [TZ]' (optional) + Returns timezone-aware UTC datetime. + """ + if not s: + return None + s = " ".join(str(s).split()) + + lower = s.lower() + if lower.startswith(("today", "yesterday")): + parts = s.split() + base = ( + datetime.now(ZoneInfo(default_tz)) + if default_tz + else datetime.now(timezone.utc) + ) + if parts[0].lower() == "yesterday": + base = base - timedelta(days=1) + + hhmm = parts[1] if len(parts) > 1 else "00:00" + tz = parts[2] if len(parts) > 2 else default_tz + if ":" in hhmm: + hh, mm = hhmm.split(":") + else: + hh, mm = hhmm[:2], hhmm[2:] + naive = base.replace(hour=int(hh), minute=int(mm), second=0, microsecond=0) + aware = ( + naive + if naive.tzinfo + else ( + naive.replace(tzinfo=ZoneInfo(tz)) + if tz + else naive.replace(tzinfo=timezone.utc) + ) ) + return aware.astimezone(timezone.utc) + + try: + dt = datetime.fromisoformat(s) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=ZoneInfo(default_tz) if default_tz else timezone.utc) + return dt.astimezone(timezone.utc) + except Exception: + pass + + try: + parts = s.split() + tz = None + if len(parts) == 3: + time_s, date_s, tz = parts + elif len(parts) == 2: + time_s, date_s = parts + else: + raise ValueError() + + if ":" in time_s: + hh, mm = time_s.split(":") + else: + hh, mm = time_s[:2], time_s[2:] + + if "/" in date_s: + mon, day, yr = date_s.split("/") + yr = int(yr) + mon = int(mon) + day = int(day) + elif "-" in date_s: + yr, mon, day = date_s.split("-") + yr = int(yr) + mon = int(mon) + day = int(day) + else: + raise ValueError() + + tzinfo = ( + ZoneInfo(tz) + if tz + else (ZoneInfo(default_tz) if default_tz else timezone.utc) + ) + local = datetime(yr, mon, day, int(hh), int(mm), tzinfo=tzinfo) + return local.astimezone(timezone.utc) + except Exception: + raise click.BadParameter(f"Invalid target_time: {s}") -# ---------- Helpers ---------- def _parse_time_or_relative(s: Optional[str]) -> Optional[datetime]: """ Accepts ISO strings (with or without tz) or relative like "24h", "3d", "90m". @@ -125,7 +239,7 @@ def _parse_time_or_relative(s: Optional[str]) -> Optional[datetime]: if not s: return None s = str(s).strip() - # relative + if s.endswith(("h", "m", "d")) and s[:-1].isdigit(): amount = int(s[:-1]) unit = s[-1] @@ -136,20 +250,12 @@ def _parse_time_or_relative(s: Optional[str]) -> Optional[datetime]: return now - timedelta(minutes=amount) if unit == "d": return now - timedelta(days=amount) - # ISO parse + try: dt = datetime.fromisoformat(s) except ValueError: raise click.BadParameter(f"Invalid datetime: {s}") - # ensure tz-aware -> UTC - if dt.tzinfo is None: - dt = dt.replace(tzinfo=timezone.utc) - return dt.astimezone(timezone.utc) - -def _ensure_tz(dt: Optional[datetime]) -> Optional[datetime]: - if dt is None: - return None if dt.tzinfo is None: dt = dt.replace(tzinfo=timezone.utc) return dt.astimezone(timezone.utc) @@ -196,28 +302,77 @@ def _fetch_multi_df( end=end, melted=True, ) - # expected columns: 'date-time', 'name', 'value' (possibly 'quality-code') - # make sure types are reasonable + if "date-time" in df.columns: df["date-time"] = pd.to_datetime(df["date-time"], utc=True, errors="coerce") return df -def _last_values_by_name(df: pd.DataFrame) -> Dict[str, float | None]: +def _window_for_target(dt: datetime, minutes: int) -> tuple[datetime, datetime]: + eps = max(1, int(minutes)) + return (dt - timedelta(minutes=eps), dt + timedelta(minutes=eps)) + + +def _expand_template(s: Optional[str], **kwargs) -> Optional[str]: + if not s: + return None + try: + return s.format(**kwargs) + except Exception: + return s + + +def _fetch_levels_dict( + level_ids: List[str], + begin: str, + end: str, + office: str, + unit: str, +) -> Dict[str, float | None]: """ - For a melted dataframe with 'name' and 'date-time' and 'value', - return the last (by time) non-null value for each name. + Return {level_id: value or None}. + We assume cwms-python supports a get_level-like call; fall back to levels endpoint if needed. """ - if df.empty: - return {} - work = df.dropna(subset=["value"]) - if work.empty: - return {} - # sort then groupby tail(1) - print(work.columns, flush=True) - work = work.sort_values(["ts_id", "date-time"]) - last = work.groupby("ts_id").tail(1) - return dict(zip(last["ts_id"], last["value"])) + out: Dict[str, float | None] = {} + for lvl in level_ids: + try: + + val = cwms.get_level_as_timeseries( + begin=datetime.fromisoformat(begin), + end=datetime.fromisoformat(end), + location_level_id=lvl, + office_id=office, + unit=unit, + ) + out[lvl] = val.json.get("values")[-1][1] if val.json.get("values") else None + except Exception as err: + print( + f"[reporting] Warning: could not fetch level '{lvl}': {err}", + traceback.format_exc(), + ) + out[lvl] = None + return out + + +def _format_value( + x: Any, + precision: Optional[int], + missing: str, + undefined: str, +) -> str: + + if x is None: + return missing + try: + xf = float(x) + if math.isnan(xf) or math.isinf(xf): + return undefined + if precision is None: + return f"{xf}" + return f"{xf:.{precision}f}" + except Exception: + + return f"{x}" def _render_template( @@ -233,19 +388,17 @@ def _render_template( import jinja2 loaders: List[jinja2.BaseLoader] = [] - # Use the built-in package templates if no user dir is given + if not template_dir: pkg_dir = os.path.abspath( os.path.join(os.path.dirname(__file__), "templates", "jinja") ) - print(pkg_dir, flush=True) if os.path.isdir(pkg_dir): loaders.append(jinja2.FileSystemLoader(pkg_dir)) - # User-specified template dir (highest priority) + if template_dir and os.path.isdir(template_dir): loaders.append(jinja2.FileSystemLoader(template_dir)) - # Fallback minimal inline template if not found env = jinja2.Environment( loader=jinja2.ChoiceLoader(loaders) if loaders else None, autoescape=True, @@ -259,100 +412,142 @@ def _render_template( tmpl = env.get_template(template_name) return tmpl.render(**context) except Exception as e: - # ultra-simple built-in fallback table - # (lets the command succeed even if no templates are set up yet) - cols = context["columns"] - rows = context["rows"] - data = context["data"] - title = f'{context.get("report",{}).get("district","") or context.get("office","") } {context.get("report",{}).get("name","Report")}' - head = ( - "Project" - + "".join(f"{c['title']}" for c in cols) - + "" - ) - body = [] - for proj in rows: - tds = [f"{proj}"] - for c in cols: - tds.append(f"{data.get(proj,{}).get(c['key'],'-')}") - body.append("" + "".join(tds) + "") - html = f""" -{title} - - -

{title}

-{head}{''.join(body)}
-""" click.echo( f"[reporting] Using built-in fallback template because '{template_name}' was not found.\nError: ({e})", err=True, ) - return html + click.echo(traceback.format_exc()) -# ---------- Main routine ---------- def build_report_table( config: Config, begin: Optional[datetime], end: Optional[datetime] ) -> Dict[str, Any]: - # rows remain as simple strings for template compatibility rows: List[str] = [p.location_id for p in config.projects] if not rows: raise click.UsageError("No 'projects' configured in YAML.") - # quick lookup proj_by_id: Dict[str, ProjectSpec] = {p.location_id: p for p in config.projects} - # columns with fallback office/unit col_defs: List[Dict[str, Any]] = [] for c in config.columns: + if not (c.tsid or c.level): + raise click.BadParameter(f"Column '{c.title}' must have 'tsid' or 'level'.") col_defs.append( { "title": c.title, - "key": c.key or c.title, + "key": c.key, "precision": c.precision, "unit": c.unit or config.default_unit, + "office": c.office or config.office, "tsid_template": c.tsid, - "office": c.office or config.office, # <- fallback here + "level_template": c.level, + "href_template": c.href, + "missing": c.missing or config.missing, + "undefined": c.undefined or config.undefined, + "target_time": c.target_time or config.target_time, } ) - # group tsids by (office, unit) - group_tsids: Dict[tuple, List[str]] = {} # (office, unit) -> tsids - backref: Dict[tuple, List[tuple]] = ( - {} - ) # (office, unit, tsid) -> [(project_id, column_key)] + base_end = end or datetime.now(timezone.utc) + + ts_groups: Dict[tuple, List[str]] = {} + backref_ts: Dict[tuple, List[tuple]] = {} + + lvl_groups: Dict[tuple, List[str]] = {} + backref_lvl: Dict[tuple, List[tuple]] = {} + + col_time_windows: Dict[str, tuple[datetime, datetime] | None] = {} + for c in col_defs: + tt = c.get("target_time") + if tt: + dt = _parse_target_like(tt, config.time_zone) + col_time_windows[c["key"]] = _window_for_target( + dt, config.time_epsilon_minutes + ) + else: + col_time_windows[c["key"]] = None for proj_id in rows: for c in col_defs: - tsid = _expand_tsid(c["tsid_template"], proj_id) - k = (c["office"], c["unit"]) - group_tsids.setdefault(k, []) - if tsid not in group_tsids[k]: - group_tsids[k].append(tsid) - backref.setdefault((c["office"], c["unit"], tsid), []).append( - (proj_id, c["key"]) - ) + office = c["office"] + unit = c["unit"] + key = c["key"] + + if c["tsid_template"]: + tsid = _expand_template(c["tsid_template"], project=proj_id) + win = col_time_windows[key] + b, e = win if win else (begin, end) + + if b is None or e is None: + e = e or base_end + b = b or (e - timedelta(hours=24)) + gk = (office, unit, b, e) + ts_groups.setdefault(gk, []) + if tsid not in ts_groups[gk]: + ts_groups[gk].append(tsid) + backref_ts.setdefault((office, unit, b, e, tsid), []).append( + (proj_id, key) + ) - # fetch & last values - last_value_by_key: Dict[tuple, float | None] = {} - for (office, unit), tsids in group_tsids.items(): + elif c["level_template"]: + lvl = _expand_template(c["level_template"], project=proj_id) + gk = (office, unit) + lvl_groups.setdefault(gk, []) + if lvl not in lvl_groups[gk]: + lvl_groups[gk].append(lvl) + backref_lvl.setdefault((office, unit, lvl), []).append((proj_id, key)) + + last_ts_value: Dict[tuple, float | None] = {} + for (office, unit, b, e), tsids in ts_groups.items(): if not tsids: continue - df = _fetch_multi_df(tsids, office, unit, begin, end) - last_vals = _last_values_by_name(df) - for ts in tsids: - last_value_by_key[(office, unit, ts)] = last_vals.get(ts) + df = _fetch_multi_df(tsids, office, unit, b, e) + + name_col = ( + "ts_id" + if "ts_id" in df.columns + else ("name" if "name" in df.columns else None) + ) + time_col = ( + "date-time" + if "date-time" in df.columns + else ("date_time" if "date_time" in df.columns else None) + ) + if name_col and time_col: + df = df.dropna(subset=[time_col]) + df[time_col] = pd.to_datetime(df[time_col], utc=True, errors="coerce") + df = df.sort_values([name_col, time_col]) + last = df.groupby(name_col).tail(1) + for _, row in last.iterrows(): + last_ts_value[(office, unit, b, e, str(row[name_col]))] = row.get( + "value", None + ) + else: + + for ts in tsids: + last_ts_value[(office, unit, b, e, ts)] = None + + last_lvl_value: Dict[tuple, float | None] = {} + for (office, unit), lvls in lvl_groups.items(): + if not lvls: + continue + vals = _fetch_levels_dict( + lvls, + begin=config.begin, + end=config.end, + office=office, + unit=unit, + ) + for lvl in lvls: + last_lvl_value[(office, unit, lvl)] = vals.get(lvl) - # prefetch project locations once and graft href proj_locations: Dict[str, Dict[str, Any]] = {} for proj_id in rows: proj = proj_by_id[proj_id] proj_office = proj.office or config.office try: loc = cwms.get_location(office_id=proj_office, location_id=proj_id) - loc_json = ( - getattr(loc, "json", None) or loc - ) # cwms-python returns object w/ .json + loc_json = getattr(loc, "json", None) or loc if isinstance(loc_json, dict): loc_json = {**loc_json} if proj.href: @@ -363,28 +558,62 @@ def build_report_table( loc_json = {"name": proj_id, "href": proj.href} proj_locations[proj_id] = loc_json - # build the table payload table: Dict[str, Dict[str, Any]] = {proj_id: {} for proj_id in rows} - for (office, unit, tsid), pairs in backref.items(): - raw_val = last_value_by_key.get((office, unit, tsid)) + for (office, unit, b, e, tsid), pairs in backref_ts.items(): + raw = last_ts_value.get((office, unit, b, e, tsid)) + for proj_id, col_key in pairs: - col = next((c for c in col_defs if c["key"] == col_key), None) - precision = col.get("precision") if col else None - table[proj_id][col_key] = _format_number(raw_val, precision) + c = next((x for x in col_defs if x["key"] == col_key), None) + val_text = _format_value( + raw, + precision=c.get("precision") if c else None, + missing=(c.get("missing") or config.missing), + undefined=(c.get("undefined") or config.undefined), + ) + href = _expand_template( + c.get("href_template"), + project=proj_id, + office=office, + tsid=tsid, + level=None, + ) + table[proj_id][col_key] = ( + {"text": val_text, "href": href} if href else {"text": val_text} + ) + + for (office, unit, lvl), pairs in backref_lvl.items(): + raw = last_lvl_value.get((office, unit, lvl)) + for proj_id, col_key in pairs: + c = next((x for x in col_defs if x["key"] == col_key), None) + val_text = _format_value( + raw, + precision=c.get("precision") if c else None, + missing=(c.get("missing") or config.missing), + undefined=(c.get("undefined") or config.undefined), + ) + href = _expand_template( + c.get("href_template"), + project=proj_id, + office=office, + tsid=None, + level=lvl, + ) + table[proj_id][col_key] = ( + {"text": val_text, "href": href} if href else {"text": val_text} + ) - # attach location block (with href) per project row for proj_id in rows: table[proj_id]["location"] = proj_locations.get(proj_id, {"name": proj_id}) return { "columns": col_defs, - "rows": rows, # still a list of project IDs for your template loop - "data": table, # data[proj]["location"]["href"] now exists (if provided) + "rows": rows, + "data": table, + "base_end": base_end, } -# ---------- Click entry ---------- @click.command( name="reporting", help="Render a CWMS timeseries report to HTML using a YAML config and Jinja2.", @@ -434,10 +663,8 @@ def build_report_table( }, ) def reporting_cli(config_path, template_dir, template_name, begin, end, out_path): - # Load config - cfg = Config.from_yaml(config_path) - # Resolve time window + cfg = Config.from_yaml(config_path) cfg_begin = ( _parse_time_or_relative(begin) if begin else _parse_time_or_relative(cfg.begin) ) @@ -445,27 +672,23 @@ def reporting_cli(config_path, template_dir, template_name, begin, end, out_path if cfg_end is None: cfg_end = datetime.now(timezone.utc) if cfg_begin is None: - # CWMS default is end-24h, but we make it explicit here cfg_begin = cfg_end - timedelta(hours=24) - # Configure cwms client API root if provided - cwms.init_session(api_root=cfg.cda_api_root) + print(f"Using time window: {cfg_begin} to {cfg_end}", flush=True) - # Build table data - print(cfg) + cwms.init_session(api_root=cfg.cda_api_root) table_ctx = build_report_table(cfg, cfg_begin, cfg_end) - # Render - base_date = cfg_end.astimezone(timezone.utc) + base_date = table_ctx.get("base_end", cfg_end).astimezone(timezone.utc) context = { "office": cfg.office, "report": dataclasses_asdict(cfg.report), "base_date": base_date, **table_ctx, } - print("TEMPLATE DIR", template_dir) html = _render_template(template_dir, template_name, context) - + if not html: + raise click.ClickException("No HTML generated.") with open(out_path, "w", encoding="utf-8") as f: f.write(html) click.echo(f"Wrote {out_path}") diff --git a/cwmscli/reporting/configs/daily.yaml b/cwmscli/reporting/configs/daily.yaml index e07f24c..b80a64b 100644 --- a/cwmscli/reporting/configs/daily.yaml +++ b/cwmscli/reporting/configs/daily.yaml @@ -1,10 +1,18 @@ -# report.yaml -office: "SWT" # fallback: $OFFICE or $CWMS_OFFICE -cda_api_root: "https://cwms-data.usace.army.mil/cwms-data/" # optional -begin: "24h" # ISO or relative like "24h", "3d", "90m" -# end: "2025-09-16T11:00:00-05:00" # optional; defaults to now (UTC) +# daily.yaml +office: "SWT" +cda_api_root: "https://cwms-data.usace.army.mil/cwms-data/" -default_unit: "EN" # can be overridden per column +# Either use a rolling window: +begin: "2025-09-16T00:00:00-05:00" +end: "2025-09-17T00:00:00-05:00" +# Or pin to a single instant, globally: +target_time: "0800 09/17/2025 America/Chicago" +time_epsilon_minutes: 10 # +/- window around target_time for TS picks; default 5 + +default_unit: "EN" +missing: "----" # used when no value at all was found +undefined: "--NA--" # used when a value exists but is NaN/invalid +time_zone: "America/Chicago" # default zone for parsing target_time if zone omitted report: district: "Tulsa District SWT" @@ -21,21 +29,26 @@ projects: href: "https://www.swt-wc.usace.army.mil/brok.lakepage.html" columns: + # Timeseries at a specific time - title: "Pool Elev (ft)" - location_id: "KEYS" - tsid: "{project}.Elev.Inst.1Hour.0.Ccp-Rev" + key: "elev_ts" office: "SWT" unit: "EN" precision: 2 - key: "elev" - - # - title: "Release (cfs)" - # tsid: "{project}.Flow-Res Out.Ave.1Hour.1Hour.0.Ccp-Rev" - # unit: "EN" - # precision: 0 - # key: "release" + tsid: "{project}.Elev.Inst.1Hour.0.Ccp-Rev" + # override the global target time for THIS column (optional): + # target_time: "2025-09-17T08:00:00-05:00" + # per-column render strings (optional): + missing: "--" + undefined: "~~~~.~~" + href: "https://water.usace.army.mil/office/{office}/ts?name={tsid}" - # - title: "Tailwater (ft)" - # tsid: "{project}.Tailwater.Inst.1Hour.0.Ccp-Rev" - # precision: 2 - # key: "tailwater" + # Level identifier (no sampling window; we just read the level) + - title: "Top of Conservation (ft)" + key: "top_cons" + office: "SWT" + unit: "ft" + precision: 2 + level: "{project}.Elev.Inst.0.Top of Conservation" + # optional per column overrides + undefined: "n/a" diff --git a/report.yaml b/report.yaml deleted file mode 100644 index 44255d5..0000000 --- a/report.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# report.yaml -office: "SWT" # fallback: $OFFICE or $CWMS_OFFICE -cda_api_root: "https://cwms-data.usace.army.mil/cwms-data" # optional -begin: "24h" # ISO or relative like "24h", "3d", "90m" -# end: "2025-09-07:00:00-05:00" # optional; defaults to now (UTC) -default_unit: "EN" # can be overridden per column - -report: - district: "Tulsa District SWT" - name: "Daily Morning Reservoir Report" - logo_left: "https://www.swt-wc.usace.army.mil/images/logos/usace-logo-color.svg" - logo_right: "https://www.swt-wc.usace.army.mil/images/logos/tulsa.png" -projects: ["KEYS", "SKIA", "BROK"] - -columns: - - title: "Pool Elev (ft)" - tsid: "{project}.Elev.Inst.1Hour.0.Ccp" # template expands {project} - unit: "EN" - precision: 2 - key: "elev" - - title: "Tailwater (ft)" - tsid: "{project}.Tailwater.Inst.1Hour.0.Ccp" - precision: 2 - - - title: "Release (cfs)" - tsid: "{project}.Flow-Res Out.Ave.1Hour.1Hour.0.Ccp-Rev" - unit: "EN" - precision: 0 - key: "release" From 84820cce2f76cf9fb8ca10fd2667df5a15db8c88 Mon Sep 17 00:00:00 2001 From: Charles Graham SWT Date: Thu, 18 Sep 2025 11:08:15 -0500 Subject: [PATCH 07/25] Make the header dynamic --- cwmscli/reporting/__init__.py | 69 +++++++- cwmscli/reporting/configs/daily.yaml | 161 ++++++++++++------ .../templates/jinja/daily/_table.html.j2 | 75 +++++--- report.html.j2 | 16 ++ 4 files changed, 238 insertions(+), 83 deletions(-) create mode 100644 report.html.j2 diff --git a/cwmscli/reporting/__init__.py b/cwmscli/reporting/__init__.py index 5c6a424..37e6eb1 100644 --- a/cwmscli/reporting/__init__.py +++ b/cwmscli/reporting/__init__.py @@ -31,6 +31,23 @@ class ReportSpec: logo_right: Optional[str] = None +@dataclass +class HeaderCellSpec: + text: str + colspan: int = 1 + rowspan: int = 1 + align: Optional[str] = None # "left"|"center"|"right" + classes: Optional[str] = None + + +@dataclass +class TableHeaderSpec: + project: HeaderCellSpec = field( + default_factory=lambda: HeaderCellSpec(text="Project", rowspan=1) + ) + rows: List[List[HeaderCellSpec]] = field(default_factory=list) + + @dataclass class ColumnSpec: title: str @@ -54,7 +71,7 @@ class Config: report: ReportSpec | Dict[str, Any] | None = None projects: List[ProjectSpec] = field(default_factory=list) columns: List[ColumnSpec] = field(default_factory=list) - + header: Optional[TableHeaderSpec] = None begin: Optional[str] = None end: Optional[str] = None @@ -122,7 +139,16 @@ def from_yaml(path: str) -> "Config": ) else: raise click.BadParameter(f"Invalid project entry: {p!r}") - + # Validate the columns and header spec + header = _parse_header_spec(raw.get("header")) + if header and header.rows: + # compute leaf-count in the final header row + leaf_count = sum(max(1, c.colspan) for c in header.rows[-1]) + if leaf_count != len(cols): + click.echo( + f"[reporting] Warning: header leaf-count ({leaf_count}) != number of data columns ({len(cols)}).", + err=True, + ) return Config( office=office, cda_api_root=raw.get("cda_api_root") or os.getenv("CDA_API_ROOT"), @@ -137,9 +163,40 @@ def from_yaml(path: str) -> "Config": missing=raw.get("missing") or "----", undefined=raw.get("undefined") or "--NA--", time_zone=raw.get("time_zone"), + header=header, ) +def _parse_header_spec(raw: Optional[Dict[str, Any]]) -> Optional["TableHeaderSpec"]: + if not raw: + return None + + def to_cell(d: Dict[str, Any]) -> HeaderCellSpec: + return HeaderCellSpec( + text=str(d.get("text", "")), + colspan=int(d.get("colspan", 1) or 1), + rowspan=int(d.get("rowspan", 1) or 1), + align=d.get("align"), + classes=d.get("classes"), + ) + + proj_raw = raw.get("project", {}) or {} + project = to_cell( + { + "text": proj_raw.get("text", "Project"), + "rowspan": proj_raw.get("rowspan", 1), + "align": proj_raw.get("align"), + "classes": proj_raw.get("classes"), + } + ) + rows_raw = raw.get("rows", []) or [] + rows = [] + for r in rows_raw: + row_cells = [to_cell(c) for c in (r or [])] + rows.append(row_cells) + return TableHeaderSpec(project=project, rows=rows) + + def _parse_target_like( s: Optional[str], default_tz: Optional[str] ) -> Optional[datetime]: @@ -405,11 +462,8 @@ def _render_template( trim_blocks=True, lstrip_blocks=True, ) - try: - if not template_name: - template_name = "report.html.j2" - tmpl = env.get_template(template_name) + tmpl = env.get_template(template_name or "report.html.j2") return tmpl.render(**context) except Exception as e: click.echo( @@ -674,8 +728,6 @@ def reporting_cli(config_path, template_dir, template_name, begin, end, out_path if cfg_begin is None: cfg_begin = cfg_end - timedelta(hours=24) - print(f"Using time window: {cfg_begin} to {cfg_end}", flush=True) - cwms.init_session(api_root=cfg.cda_api_root) table_ctx = build_report_table(cfg, cfg_begin, cfg_end) @@ -684,6 +736,7 @@ def reporting_cli(config_path, template_dir, template_name, begin, end, out_path "office": cfg.office, "report": dataclasses_asdict(cfg.report), "base_date": base_date, + "header": dataclasses_asdict(cfg.header), **table_ctx, } html = _render_template(template_dir, template_name, context) diff --git a/cwmscli/reporting/configs/daily.yaml b/cwmscli/reporting/configs/daily.yaml index b80a64b..d291809 100644 --- a/cwmscli/reporting/configs/daily.yaml +++ b/cwmscli/reporting/configs/daily.yaml @@ -1,54 +1,107 @@ -# daily.yaml -office: "SWT" -cda_api_root: "https://cwms-data.usace.army.mil/cwms-data/" - -# Either use a rolling window: -begin: "2025-09-16T00:00:00-05:00" -end: "2025-09-17T00:00:00-05:00" -# Or pin to a single instant, globally: -target_time: "0800 09/17/2025 America/Chicago" -time_epsilon_minutes: 10 # +/- window around target_time for TS picks; default 5 - -default_unit: "EN" -missing: "----" # used when no value at all was found -undefined: "--NA--" # used when a value exists but is NaN/invalid -time_zone: "America/Chicago" # default zone for parsing target_time if zone omitted - -report: - district: "Tulsa District SWT" - name: "Daily Morning Reservoir Report" - logo_left: "https://www.swt-wc.usace.army.mil/images/logos/usace-logo-color.svg" - logo_right: "https://www.swt-wc.usace.army.mil/images/logos/tulsa.png" - -projects: - - location_id: "KEYS" - href: "https://www.swt-wc.usace.army.mil/keys.lakepage.html" - - location_id: "SKIA" - href: "https://www.swt-wc.usace.army.mil/skia.lakepage.html" - - location_id: "BROK" - href: "https://www.swt-wc.usace.army.mil/brok.lakepage.html" - -columns: - # Timeseries at a specific time - - title: "Pool Elev (ft)" - key: "elev_ts" - office: "SWT" - unit: "EN" - precision: 2 - tsid: "{project}.Elev.Inst.1Hour.0.Ccp-Rev" - # override the global target time for THIS column (optional): - # target_time: "2025-09-17T08:00:00-05:00" - # per-column render strings (optional): - missing: "--" - undefined: "~~~~.~~" - href: "https://water.usace.army.mil/office/{office}/ts?name={tsid}" - - # Level identifier (no sampling window; we just read the level) - - title: "Top of Conservation (ft)" - key: "top_cons" - office: "SWT" - unit: "ft" - precision: 2 - level: "{project}.Elev.Inst.0.Top of Conservation" - # optional per column overrides - undefined: "n/a" +# daily.yaml +office: "SWT" +cda_api_root: "https://cwms-data.usace.army.mil/cwms-data/" + +# Either use a rolling window: +begin: "2025-09-16T00:00:00-05:00" +end: "2025-09-17T00:00:00-05:00" +# Or pin to a single instant, globally: +target_time: "0800 09/17/2025 America/Chicago" +time_epsilon_minutes: 10 # +/- window around target_time for TS picks; default 5 + +default_unit: "EN" +missing: "----" # used when no value at all was found +undefined: "--NA--" # used when a value exists but is NaN/invalid +time_zone: "America/Chicago" # default zone for parsing target_time if zone omitted + +report: + district: "Tulsa District SWT" + name: "Daily Morning Reservoir Report" + logo_left: "https://www.swt-wc.usace.army.mil/images/logos/usace-logo-color.svg" + logo_right: "https://www.swt-wc.usace.army.mil/images/logos/tulsa.png" + +projects: + - location_id: "KEYS" + href: "https://www.swt-wc.usace.army.mil/keys.lakepage.html" + - location_id: "SKIA" + href: "https://www.swt-wc.usace.army.mil/skia.lakepage.html" + - location_id: "BROK" + href: "https://www.swt-wc.usace.army.mil/brok.lakepage.html" + +header: + project: + text: "Reservoir" # text in the far-left header cell + classes: "" # optional extra classes + align: center # optional: left|center|right + rows: + # Header row 1 + - - + text: "Pool Elevation" + colspan: 2 + - text: "Pool Limits" + colspan: 2 + - text: "Pool Occupied Storage" + colspan: 3 + - text: "8AM Status (cfs)" + colspan: 3 + - text: "Previous 24Hr Average (cfs)" + colspan: 3 + - text: "Precip (in)" + align: center + # Header row 2 + - - text: "" + rowspan: 2 + - text: "8AM
Current" + rowspan: 2 + - text: "8AM
Prev" + rowspan: 2 + - text: "Top Of" + colspan: 2 + - text: "(%)" + rowspan: 2 + - text: "C/F" + rowspan: 2 + - text: "(ac-ft)" + rowspan: 2 + - text: "Total Release" + rowspan: 2 + - text: "Power Release" + rowspan: 2 + - text: "Inflow" + rowspan: 2 + - text: "Total Release" + rowspan: 2 + - text: "Power Release" + rowspan: 2 + - text: "Inflow" + rowspan: 2 + - text: "24Hr" + rowspan: 2 + # Header row 3 + - - text: "Pool Elev (ft)" + - text: "Tailwater (ft)" + +columns: + # Timeseries at a specific time + - title: "Pool Elev (ft)" + key: "elev_ts" + office: "SWT" + unit: "EN" + precision: 2 + tsid: "{project}.Elev.Inst.1Hour.0.Ccp-Rev" + # override the global target time for THIS column (optional): + target_time: "2025-09-17T08:00:00-05:00" + # per-column render strings (optional): + missing: "--" + undefined: "~~~~.~~" + href: "https://water.usace.army.mil/office/{office}/ts?name={tsid}" + + # Level identifier (no sampling window; we just read the level) + - title: "Top of Conservation (ft)" + key: "top_cons" + office: "SWT" + unit: "ft" + precision: 2 + level: "{project}.Elev.Inst.0.Top of Conservation" + # optional per column overrides + undefined: "n/a" diff --git a/cwmscli/reporting/templates/jinja/daily/_table.html.j2 b/cwmscli/reporting/templates/jinja/daily/_table.html.j2 index b0f8eba..565e9a2 100644 --- a/cwmscli/reporting/templates/jinja/daily/_table.html.j2 +++ b/cwmscli/reporting/templates/jinja/daily/_table.html.j2 @@ -1,23 +1,56 @@
- - - - - {% for col in columns %} - - {% endfor %} - - - - {% for proj in rows %} - - - {% for col in columns %} - - {% endfor %} - - {% endfor %} - -
Project{{ col.title }}
- {{data[proj]["location"]["public-name"]}}{{ data[proj][col.key] }}
+ + + {% if header and header.rows and header.rows|length > 0 %} + {% set header_row_count = header.rows|length %} + {% for r in header.rows %} + + {% if loop.first %} + {% set pr = header.project %} + + {% endif %} + + {% for cell in r %} + + {% endfor %} + + {% endfor %} + {% else %} + + + {% for col in columns %} + + {% endfor %} + + {% endif %} + + + + + {% for proj in rows %} + + + {% for col in columns %} + {% set cell = data[proj][col.key] %} + + {% endfor %} + + {% endfor %} + +
{{ pr.text | safe }} 1 %}colspan="{{ cell.colspan }}"{% endif %} + {% if cell.rowspan and cell.rowspan > 1 %}rowspan="{{ cell.rowspan }}"{% endif %} + class="{{ cell.classes or '' }}" + {% if cell.align %}style="text-align: {{ cell.align }};"{% endif %} + >{{ cell.text | safe }}
{{ (header.project.text if header else "Project") | safe }}{{ col.title | safe }}
+ + {{ data[proj]["location"]["public-name"] or proj | safe }} + + + {% if cell.href %} + {{ cell.text | safe }} + {% else %} + {{ cell.text | safe }} + {% endif %} +
\ No newline at end of file diff --git a/report.html.j2 b/report.html.j2 new file mode 100644 index 0000000..d8576be --- /dev/null +++ b/report.html.j2 @@ -0,0 +1,16 @@ + + + {{ title }} + +

{{ title }}

+

Office: {{ office }}

+

Generated at: {{ generated_at }}

+ +

Latest values

+
    + {% for item in series %} +
  • {{ item.name }}: {{ item.value }} {{ item.unit }} at {{ item.time }}
  • + {% endfor %} +
+ + From d724fc1dac6ffc12dfad1d2d8f5d37dc0ae533fc Mon Sep 17 00:00:00 2001 From: Charles Graham SWT Date: Thu, 18 Sep 2025 11:09:20 -0500 Subject: [PATCH 08/25] Add reporting diagram --- cwmscli/reporting/reporting_plan.excalidraw | 943 ++++++++++++++++++++ 1 file changed, 943 insertions(+) create mode 100644 cwmscli/reporting/reporting_plan.excalidraw diff --git a/cwmscli/reporting/reporting_plan.excalidraw b/cwmscli/reporting/reporting_plan.excalidraw new file mode 100644 index 0000000..8a5ca69 --- /dev/null +++ b/cwmscli/reporting/reporting_plan.excalidraw @@ -0,0 +1,943 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "id": "45BrgjiDTRbrt9oFnGnfe", + "type": "line", + "x": 1667.3333740234375, + "y": 262.3333435058594, + "width": 4.6666259765625, + "height": 642.0000305175781, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "a0", + "roundness": { + "type": 2 + }, + "seed": 1305822094, + "version": 60, + "versionNonce": 1311864590, + "isDeleted": false, + "boundElements": [], + "updated": 1757708266572, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 4.6666259765625, + 642.0000305175781 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null, + "polygon": false + }, + { + "id": "B3qzuBdq_DVvL5wwmQpYQ", + "type": "line", + "x": 1293.3333740234375, + "y": 261, + "width": 11.3333740234375, + "height": 650, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "a2", + "roundness": { + "type": 2 + }, + "seed": 1570201550, + "version": 44, + "versionNonce": 983650638, + "isDeleted": false, + "boundElements": [], + "updated": 1757708272473, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 11.3333740234375, + 650 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null, + "polygon": false + }, + { + "id": "kuGIke4Czdp_dF3qFH338", + "type": "line", + "x": 844.6666870117188, + "y": 235, + "width": 18.66668701171875, + "height": 710.666748046875, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "a4", + "roundness": { + "type": 2 + }, + "seed": 1367578894, + "version": 39, + "versionNonce": 1225538578, + "isDeleted": false, + "boundElements": [], + "updated": 1757708277840, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 18.66668701171875, + 710.666748046875 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null, + "polygon": false + }, + { + "id": "SZ5fGPXcHB-x6WdqQaHCi", + "type": "line", + "x": 382.66668701171875, + "y": 227.66668701171875, + "width": 8, + "height": 727.3333129882812, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "a5", + "roundness": { + "type": 2 + }, + "seed": 1442176914, + "version": 18, + "versionNonce": 1005071186, + "isDeleted": false, + "boundElements": [], + "updated": 1757708279690, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 8, + 727.3333129882812 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null, + "polygon": false + }, + { + "id": "RXg6zUDe6Bo-g-NivAepi", + "type": "text", + "x": 1836, + "y": 531, + "width": 46.839996337890625, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "a6", + "roundness": null, + "seed": 1133444818, + "version": 5, + "versionNonce": 528186258, + "isDeleted": false, + "boundElements": [], + "updated": 1757708282892, + "link": null, + "locked": false, + "text": "TXT", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "TXT", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "B0eSaFa5tT1unoyJsHEWu", + "type": "text", + "x": 1359.3333740234375, + "y": 529.6666870117188, + "width": 198.9598388671875, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "a7", + "roundness": null, + "seed": 828994318, + "version": 64, + "versionNonce": 871114702, + "isDeleted": false, + "boundElements": [], + "updated": 1757945731090, + "link": null, + "locked": false, + "text": "URLLib2 / Requests", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "URLLib2 / Requests", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "jfbjdxIaMwa5pF2-bVjL3", + "type": "text", + "x": 940.6666870117188, + "y": 468.3333740234375, + "width": 98.95993041992188, + "height": 100, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "a8", + "roundness": null, + "seed": 1378989646, + "version": 28, + "versionNonce": 1096723406, + "isDeleted": false, + "boundElements": [], + "updated": 1757709026354, + "link": null, + "locked": false, + "text": "Templates\nForm\n\nJinja2", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Templates\nForm\n\nJinja2", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "E4EIyz-6lRInFCfOU8Mxt", + "type": "rectangle", + "x": 451.3333435058594, + "y": 361.66668701171875, + "width": 343.3333435058594, + "height": 343.33331298828125, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "a9", + "roundness": { + "type": 3 + }, + "seed": 1215501646, + "version": 48, + "versionNonce": 1664807246, + "isDeleted": false, + "boundElements": [ + { + "id": "K5ajuQAFI2A5Zj27sJMf3", + "type": "arrow" + } + ], + "updated": 1757708355873, + "link": null, + "locked": false + }, + { + "id": "EwP1a0nvPkBE_JUabswiE", + "type": "rectangle", + "x": 485.3333435058594, + "y": 413, + "width": 210.00003051757812, + "height": 168, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aA", + "roundness": { + "type": 3 + }, + "seed": 1018672722, + "version": 67, + "versionNonce": 1400516110, + "isDeleted": false, + "boundElements": [], + "updated": 1757708333527, + "link": null, + "locked": false + }, + { + "id": "3rcUcb5_ovR6TqjpQe4TV", + "type": "rectangle", + "x": 706, + "y": 396.3333740234375, + "width": 80, + "height": 194.6666259765625, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aB", + "roundness": { + "type": 3 + }, + "seed": 559186574, + "version": 60, + "versionNonce": 1651013134, + "isDeleted": false, + "boundElements": [], + "updated": 1757708337127, + "link": null, + "locked": false + }, + { + "id": "cyV2yj1e7QARe2_uJlkyL", + "type": "rectangle", + "x": 522, + "y": 466.3333740234375, + "width": 64, + "height": 35, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aC", + "roundness": { + "type": 3 + }, + "seed": 1867565710, + "version": 143, + "versionNonce": 1667998862, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "MNuYyHSUjkT4svn2wrR93" + } + ], + "updated": 1757708431200, + "link": null, + "locked": false + }, + { + "id": "MNuYyHSUjkT4svn2wrR93", + "type": "text", + "x": 539.2100067138672, + "y": 471.3333740234375, + "width": 29.579986572265625, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aD", + "roundness": null, + "seed": 1661873166, + "version": 110, + "versionNonce": 1206117070, + "isDeleted": false, + "boundElements": [], + "updated": 1757708431200, + "link": null, + "locked": false, + "text": "TS", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "cyV2yj1e7QARe2_uJlkyL", + "originalText": "TS", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "wWR0namAJs_EjNYuACQ6R", + "type": "rectangle", + "x": 604, + "y": 466.83331298828125, + "width": 64, + "height": 35, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aE", + "roundness": { + "type": 3 + }, + "seed": 559258830, + "version": 233, + "versionNonce": 358573586, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "ttYbn-SQ2hiJ0a8O2azKg" + } + ], + "updated": 1757708433117, + "link": null, + "locked": false + }, + { + "id": "ttYbn-SQ2hiJ0a8O2azKg", + "type": "text", + "x": 621.2100067138672, + "y": 471.83331298828125, + "width": 29.579986572265625, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aF", + "roundness": null, + "seed": 1583665934, + "version": 200, + "versionNonce": 74240978, + "isDeleted": false, + "boundElements": [], + "updated": 1757708433117, + "link": null, + "locked": false, + "text": "TS", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "wWR0namAJs_EjNYuACQ6R", + "originalText": "TS", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "K5ajuQAFI2A5Zj27sJMf3", + "type": "arrow", + "x": 801.3333740234375, + "y": 388.3333435058594, + "width": 212, + "height": 1.333343505859375, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aG", + "roundness": { + "type": 2 + }, + "seed": 1864957902, + "version": 42, + "versionNonce": 1277330190, + "isDeleted": false, + "boundElements": [], + "updated": 1757708355873, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 212, + 1.333343505859375 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "E4EIyz-6lRInFCfOU8Mxt", + "focus": -0.8458738461891708, + "gap": 6.88217589642898 + }, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "0cbaVcjKa1vrn60B_IuNx", + "type": "rectangle", + "x": 478.66668701171875, + "y": 630.3333740234375, + "width": 101.33331298828125, + "height": 40.6666259765625, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aH", + "roundness": { + "type": 3 + }, + "seed": 643402766, + "version": 44, + "versionNonce": 992353294, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "zPrZuXPWf-Ma2erGqn5I7" + } + ], + "updated": 1757708381095, + "link": null, + "locked": false + }, + { + "id": "zPrZuXPWf-Ma2erGqn5I7", + "type": "text", + "x": 497.4833679199219, + "y": 638.1666870117188, + "width": 63.699951171875, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aHV", + "roundness": null, + "seed": 1614448146, + "version": 16, + "versionNonce": 2019762706, + "isDeleted": false, + "boundElements": [], + "updated": 1757708384700, + "link": null, + "locked": false, + "text": "Submit", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "0cbaVcjKa1vrn60B_IuNx", + "originalText": "Submit", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "_0KuYh5mric3SfyJ03YLL", + "type": "rectangle", + "x": 600.6666870117188, + "y": 624.3333740234375, + "width": 133.33331298828125, + "height": 54, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aI", + "roundness": { + "type": 3 + }, + "seed": 87026638, + "version": 38, + "versionNonce": 1546813198, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "nKEXMZBE19WnimhaBoSDy" + }, + { + "id": "OuEZ50H_nbxcYXCTIiu1F", + "type": "arrow" + } + ], + "updated": 1757708390351, + "link": null, + "locked": false + }, + { + "id": "nKEXMZBE19WnimhaBoSDy", + "type": "text", + "x": 638.4533615112305, + "y": 638.8333740234375, + "width": 57.75996398925781, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aJ", + "roundness": null, + "seed": 436072206, + "version": 16, + "versionNonce": 1471615758, + "isDeleted": false, + "boundElements": [], + "updated": 1757708378813, + "link": null, + "locked": false, + "text": "Hourly", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "_0KuYh5mric3SfyJ03YLL", + "originalText": "Hourly", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "OuEZ50H_nbxcYXCTIiu1F", + "type": "arrow", + "x": 745.3333740234375, + "y": 647, + "width": 244, + "height": 1.33331298828125, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aK", + "roundness": { + "type": 2 + }, + "seed": 606278798, + "version": 50, + "versionNonce": 1382598862, + "isDeleted": false, + "boundElements": [], + "updated": 1757708390351, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 244, + -1.33331298828125 + ] + ], + "lastCommittedPoint": null, + "startBinding": { + "elementId": "_0KuYh5mric3SfyJ03YLL", + "focus": -0.14278281687867803, + "gap": 11.3333740234375 + }, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "elbowed": false + }, + { + "id": "GFFseBSOQgL96aadubyK8", + "type": "rectangle", + "x": 488.66668701171875, + "y": 415.66668701171875, + "width": 197.33331298828125, + "height": 45.33331298828125, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aL", + "roundness": { + "type": 3 + }, + "seed": 409175890, + "version": 35, + "versionNonce": 519524370, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "lRAL796I7x61SLOhtQC_P" + } + ], + "updated": 1757708437692, + "link": null, + "locked": false + }, + { + "id": "lRAL796I7x61SLOhtQC_P", + "type": "text", + "x": 554.9333724975586, + "y": 425.8333435058594, + "width": 64.79994201660156, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aM", + "roundness": null, + "seed": 1962646478, + "version": 8, + "versionNonce": 2078830030, + "isDeleted": false, + "boundElements": [], + "updated": 1757708439148, + "link": null, + "locked": false, + "text": "Header", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "GFFseBSOQgL96aadubyK8", + "originalText": "Header", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "VcPgwfeNYGhBMMseONuJ1", + "type": "text", + "x": 486.66668701171875, + "y": 235, + "width": 96.91990661621094, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "am", + "roundness": null, + "seed": 735510286, + "version": 11, + "versionNonce": 345654222, + "isDeleted": false, + "boundElements": [], + "updated": 1757708968300, + "link": null, + "locked": false, + "text": "Web Layer", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Web Layer", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "I32-JoML08GPd6X8kCeFB", + "type": "text", + "x": 967.3333740234375, + "y": 241, + "width": 203.0398712158203, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "an", + "roundness": null, + "seed": 2008335566, + "version": 24, + "versionNonce": 1667685778, + "isDeleted": false, + "boundElements": [], + "updated": 1757708999092, + "link": null, + "locked": false, + "text": "CLI + Template layer", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "CLI + Template layer", + "autoResize": true, + "lineHeight": 1.25 + } + ], + "appState": { + "gridSize": 20, + "gridStep": 5, + "gridModeEnabled": false, + "viewBackgroundColor": "#ffffff", + "lockedMultiSelections": {} + }, + "files": {} +} \ No newline at end of file From 713e14c998f39cd96da1e8fd90da4a9bb87ec722 Mon Sep 17 00:00:00 2001 From: Charles Graham SWT Date: Mon, 22 Sep 2025 16:49:01 -0500 Subject: [PATCH 09/25] Add additional location placeholders for report, test target_time --- cwmscli/reporting/configs/daily.yaml | 115 ++++++++++++++++++++++++--- 1 file changed, 106 insertions(+), 9 deletions(-) diff --git a/cwmscli/reporting/configs/daily.yaml b/cwmscli/reporting/configs/daily.yaml index d291809..eebff1f 100644 --- a/cwmscli/reporting/configs/daily.yaml +++ b/cwmscli/reporting/configs/daily.yaml @@ -21,16 +21,102 @@ report: logo_right: "https://www.swt-wc.usace.army.mil/images/logos/tulsa.png" projects: - - location_id: "KEYS" - href: "https://www.swt-wc.usace.army.mil/keys.lakepage.html" - - location_id: "SKIA" - href: "https://www.swt-wc.usace.army.mil/skia.lakepage.html" - - location_id: "BROK" - href: "https://www.swt-wc.usace.army.mil/brok.lakepage.html" + # - location_id: "SKIA" + # href: "https://www.swt-wc.usace.army.mil/skia.lakepage.html" + # - location_id: "BROK" + # href: "https://www.swt-wc.usace.army.mil/brok.lakepage.html" + # - location_id: "ALTU" + # href: "https://www.swt-wc.usace.army.mil/ALTU.lakepage.html" + # - location_id: "ARBU" + # href: "https://www.swt-wc.usace.army.mil/ARBU.lakepage.html" + # - location_id: "ARCA" + # href: "https://www.swt-wc.usace.army.mil/ARCA.lakepage.html" + # - location_id: "BIGH" + # href: "https://www.swt-wc.usace.army.mil/BIGH.lakepage.html" + # - location_id: "BIRC" + # href: "https://www.swt-wc.usace.army.mil/BIRC.lakepage.html" + # - location_id: "BROK" + # href: "https://www.swt-wc.usace.army.mil/BROK.lakepage.html" + # - location_id: "CANT" + # href: "https://www.swt-wc.usace.army.mil/CANT.lakepage.html" + # - location_id: "CHEN" + # href: "https://www.swt-wc.usace.army.mil/CHEN.lakepage.html" + # - location_id: "FCOB" + # href: "https://www.swt-wc.usace.army.mil/FCOB.lakepage.html" + # - location_id: "COPA" + # href: "https://www.swt-wc.usace.army.mil/COPA.lakepage.html" + # - location_id: "COUN" + # href: "https://www.swt-wc.usace.army.mil/COUN.lakepage.html" + # - location_id: "DENI" + # href: "https://www.swt-wc.usace.army.mil/DENI.lakepage.html" + # - location_id: "ELDR" + # href: "https://www.swt-wc.usace.army.mil/ELDR.lakepage.html" + # - location_id: "ELKC" + # href: "https://www.swt-wc.usace.army.mil/ELKC.lakepage.html" + # - location_id: "EUFA" + # href: "https://www.swt-wc.usace.army.mil/EUFA.lakepage.html" + # - location_id: "FALL" + # href: "https://www.swt-wc.usace.army.mil/FALL.lakepage.html" + # - location_id: "FCOB" + # href: "https://www.swt-wc.usace.army.mil/FCOB.lakepage.html" + - location_id: "FGIB" + href: "https://www.swt-wc.usace.army.mil/FGIB.lakepage.html" + # - location_id: "FOSS" + # href: "https://www.swt-wc.usace.army.mil/FOSS.lakepage.html" + # - location_id: "FSUP" + # href: "https://www.swt-wc.usace.army.mil/FSUP.lakepage.html" + # - location_id: "GSAL" + # href: "https://www.swt-wc.usace.army.mil/GSAL.lakepage.html" + # - location_id: "HEYB" + # href: "https://www.swt-wc.usace.army.mil/HEYB.lakepage.html" + # - location_id: "HUDS" + # href: "https://www.swt-wc.usace.army.mil/HUDS.lakepage.html" + # - location_id: "HUGO" + # href: "https://www.swt-wc.usace.army.mil/HUGO.lakepage.html" + # - location_id: "HULA" + # href: "https://www.swt-wc.usace.army.mil/HULA.lakepage.html" + # - location_id: "JOHN" + # href: "https://www.swt-wc.usace.army.mil/JOHN.lakepage.html" + # - location_id: "KAWL" + # href: "https://www.swt-wc.usace.army.mil/KAWL.lakepage.html" + # - location_id: "KEMP" + # href: "https://www.swt-wc.usace.army.mil/KEMP.lakepage.html" + # - location_id: "KEYS" + # href: "https://www.swt-wc.usace.army.mil/KEYS.lakepage.html" + # - location_id: "MARI" + # href: "https://www.swt-wc.usace.army.mil/MARI.lakepage.html" + # - location_id: "MCGE" + # href: "https://www.swt-wc.usace.army.mil/MCGE.lakepage.html" + # - location_id: "MERE" + # href: "https://www.swt-wc.usace.army.mil/MERE.lakepage.html" + # - location_id: "OOLO" + # href: "https://www.swt-wc.usace.army.mil/OOLO.lakepage.html" + # - location_id: "PATM" + # href: "https://www.swt-wc.usace.army.mil/PATM.lakepage.html" + # - location_id: "PENS" + # href: "https://www.swt-wc.usace.army.mil/PENS.lakepage.html" + # - location_id: "PINE" + # href: "https://www.swt-wc.usace.army.mil/PINE.lakepage.html" + # - location_id: "SARD" + # href: "https://www.swt-wc.usace.army.mil/SARD.lakepage.html" + # - location_id: "SKIA" + # href: "https://www.swt-wc.usace.army.mil/SKIA.lakepage.html" + # - location_id: "TENK" + # href: "https://www.swt-wc.usace.army.mil/TENK.lakepage.html" + # - location_id: "THUN" + # href: "https://www.swt-wc.usace.army.mil/THUN.lakepage.html" + # - location_id: "TOMS" + # href: "https://www.swt-wc.usace.army.mil/TOMS.lakepage.html" + # - location_id: "TORO" + # href: "https://www.swt-wc.usace.army.mil/TORO.lakepage.html" + # - location_id: "WAUR" + # href: "https://www.swt-wc.usace.army.mil/WAUR.lakepage.html" + # - location_id: "WIST" + # href: "https://www.swt-wc.usace.army.mil/WIST.lakepage.html" header: project: - text: "Reservoir" # text in the far-left header cell + text: "Reservoir" # text in the far-left header cell classes: "" # optional extra classes align: center # optional: left|center|right rows: @@ -90,12 +176,23 @@ columns: precision: 2 tsid: "{project}.Elev.Inst.1Hour.0.Ccp-Rev" # override the global target time for THIS column (optional): - target_time: "2025-09-17T08:00:00-05:00" + target_time: "2025-09-22T08:00:00-05:00" + # per-column render strings (optional): + missing: "--" + undefined: "~~~~.~~" + href: "https://water.usace.army.mil/office/{office}/ts?name={tsid}" + - title: "Pool Elev (ft)" + key: "elev_ts" + office: "SWT" + unit: "EN" + precision: 2 + tsid: "{project}.Elev.Inst.1Hour.0.Ccp-Rev" + # override the global target time for THIS column (optional): + target_time: "2025-09-21T08:00:00-05:00" # per-column render strings (optional): missing: "--" undefined: "~~~~.~~" href: "https://water.usace.army.mil/office/{office}/ts?name={tsid}" - # Level identifier (no sampling window; we just read the level) - title: "Top of Conservation (ft)" key: "top_cons" From 2874fc1096ebd364f4893969590d61bb0fc87df3 Mon Sep 17 00:00:00 2001 From: Charles Graham SWT Date: Tue, 23 Sep 2025 09:36:38 -0500 Subject: [PATCH 10/25] Add dateparser/dateutil for extensive date options/strings in cwms-cli --- poetry.lock | 182 ++++++++++++++++++++++++++++++++++++++++++++++--- pyproject.toml | 2 + 2 files changed, 176 insertions(+), 8 deletions(-) diff --git a/poetry.lock b/poetry.lock index 692853c..2ba922a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -205,6 +205,29 @@ pandas = ">=2.1.3,<3.0.0" requests = ">=2.31.0,<3.0.0" requests-toolbelt = ">=1.0.0,<2.0.0" +[[package]] +name = "dateparser" +version = "1.2.0" +description = "Date parsing library designed to parse dates from HTML pages" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "dateparser-1.2.0-py2.py3-none-any.whl", hash = "sha256:0b21ad96534e562920a0083e97fd45fa959882d4162acc358705144520a35830"}, + {file = "dateparser-1.2.0.tar.gz", hash = "sha256:7975b43a4222283e0ae15be7b4999d08c9a70e2d378ac87385b1ccf2cffbbb30"}, +] + +[package.dependencies] +python-dateutil = "*" +pytz = "*" +regex = "<2019.02.19 || >2019.02.19,<2021.8.27 || >2021.8.27" +tzlocal = "*" + +[package.extras] +calendars = ["convertdate", "hijri-converter"] +fasttext = ["fasttext"] +langdetect = ["langdetect"] + [[package]] name = "distlib" version = "0.4.0" @@ -761,14 +784,14 @@ email = ["email-validator (>=1.0.3)"] [[package]] name = "python-dateutil" -version = "2.9.0.post0" +version = "2.9.0" description = "Extensions to the standard Python datetime module" -optional = true +optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] files = [ - {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, - {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, + {file = "python-dateutil-2.9.0.tar.gz", hash = "sha256:78e73e19c63f5b20ffa567001531680d939dc042bf7850431877645523c66709"}, + {file = "python_dateutil-2.9.0-py2.py3-none-any.whl", hash = "sha256:cbf2f1da5e6083ac2fbfd4da39a25f34312230110440f424a14c7558bb85d82e"}, ] [package.dependencies] @@ -778,7 +801,7 @@ six = ">=1.5" name = "pytz" version = "2025.2" description = "World timezone definitions, modern and historical" -optional = true +optional = false python-versions = "*" groups = ["main"] files = [ @@ -849,6 +872,131 @@ files = [ {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] +[[package]] +name = "regex" +version = "2025.9.18" +description = "Alternative regular expression module, to replace re." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "regex-2025.9.18-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:12296202480c201c98a84aecc4d210592b2f55e200a1d193235c4db92b9f6788"}, + {file = "regex-2025.9.18-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:220381f1464a581f2ea988f2220cf2a67927adcef107d47d6897ba5a2f6d51a4"}, + {file = "regex-2025.9.18-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:87f681bfca84ebd265278b5daa1dcb57f4db315da3b5d044add7c30c10442e61"}, + {file = "regex-2025.9.18-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34d674cbba70c9398074c8a1fcc1a79739d65d1105de2a3c695e2b05ea728251"}, + {file = "regex-2025.9.18-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:385c9b769655cb65ea40b6eea6ff763cbb6d69b3ffef0b0db8208e1833d4e746"}, + {file = "regex-2025.9.18-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8900b3208e022570ae34328712bef6696de0804c122933414014bae791437ab2"}, + {file = "regex-2025.9.18-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c204e93bf32cd7a77151d44b05eb36f469d0898e3fba141c026a26b79d9914a0"}, + {file = "regex-2025.9.18-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3acc471d1dd7e5ff82e6cacb3b286750decd949ecd4ae258696d04f019817ef8"}, + {file = "regex-2025.9.18-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6479d5555122433728760e5f29edb4c2b79655a8deb681a141beb5c8a025baea"}, + {file = "regex-2025.9.18-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:431bd2a8726b000eb6f12429c9b438a24062a535d06783a93d2bcbad3698f8a8"}, + {file = "regex-2025.9.18-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0cc3521060162d02bd36927e20690129200e5ac9d2c6d32b70368870b122db25"}, + {file = "regex-2025.9.18-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a021217b01be2d51632ce056d7a837d3fa37c543ede36e39d14063176a26ae29"}, + {file = "regex-2025.9.18-cp310-cp310-win32.whl", hash = "sha256:4a12a06c268a629cb67cc1d009b7bb0be43e289d00d5111f86a2efd3b1949444"}, + {file = "regex-2025.9.18-cp310-cp310-win_amd64.whl", hash = "sha256:47acd811589301298c49db2c56bde4f9308d6396da92daf99cba781fa74aa450"}, + {file = "regex-2025.9.18-cp310-cp310-win_arm64.whl", hash = "sha256:16bd2944e77522275e5ee36f867e19995bcaa533dcb516753a26726ac7285442"}, + {file = "regex-2025.9.18-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:51076980cd08cd13c88eb7365427ae27f0d94e7cebe9ceb2bb9ffdae8fc4d82a"}, + {file = "regex-2025.9.18-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:828446870bd7dee4e0cbeed767f07961aa07f0ea3129f38b3ccecebc9742e0b8"}, + {file = "regex-2025.9.18-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c28821d5637866479ec4cc23b8c990f5bc6dd24e5e4384ba4a11d38a526e1414"}, + {file = "regex-2025.9.18-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:726177ade8e481db669e76bf99de0b278783be8acd11cef71165327abd1f170a"}, + {file = "regex-2025.9.18-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5cca697da89b9f8ea44115ce3130f6c54c22f541943ac8e9900461edc2b8bd4"}, + {file = "regex-2025.9.18-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dfbde38f38004703c35666a1e1c088b778e35d55348da2b7b278914491698d6a"}, + {file = "regex-2025.9.18-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f2f422214a03fab16bfa495cfec72bee4aaa5731843b771860a471282f1bf74f"}, + {file = "regex-2025.9.18-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a295916890f4df0902e4286bc7223ee7f9e925daa6dcdec4192364255b70561a"}, + {file = "regex-2025.9.18-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:5db95ff632dbabc8c38c4e82bf545ab78d902e81160e6e455598014f0abe66b9"}, + {file = "regex-2025.9.18-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fb967eb441b0f15ae610b7069bdb760b929f267efbf522e814bbbfffdf125ce2"}, + {file = "regex-2025.9.18-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f04d2f20da4053d96c08f7fde6e1419b7ec9dbcee89c96e3d731fca77f411b95"}, + {file = "regex-2025.9.18-cp311-cp311-win32.whl", hash = "sha256:895197241fccf18c0cea7550c80e75f185b8bd55b6924fcae269a1a92c614a07"}, + {file = "regex-2025.9.18-cp311-cp311-win_amd64.whl", hash = "sha256:7e2b414deae99166e22c005e154a5513ac31493db178d8aec92b3269c9cce8c9"}, + {file = "regex-2025.9.18-cp311-cp311-win_arm64.whl", hash = "sha256:fb137ec7c5c54f34a25ff9b31f6b7b0c2757be80176435bf367111e3f71d72df"}, + {file = "regex-2025.9.18-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:436e1b31d7efd4dcd52091d076482031c611dde58bf9c46ca6d0a26e33053a7e"}, + {file = "regex-2025.9.18-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c190af81e5576b9c5fdc708f781a52ff20f8b96386c6e2e0557a78402b029f4a"}, + {file = "regex-2025.9.18-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e4121f1ce2b2b5eec4b397cc1b277686e577e658d8f5870b7eb2d726bd2300ab"}, + {file = "regex-2025.9.18-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:300e25dbbf8299d87205e821a201057f2ef9aa3deb29caa01cd2cac669e508d5"}, + {file = "regex-2025.9.18-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7b47fcf9f5316c0bdaf449e879407e1b9937a23c3b369135ca94ebc8d74b1742"}, + {file = "regex-2025.9.18-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:57a161bd3acaa4b513220b49949b07e252165e6b6dc910ee7617a37ff4f5b425"}, + {file = "regex-2025.9.18-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f130c3a7845ba42de42f380fff3c8aebe89a810747d91bcf56d40a069f15352"}, + {file = "regex-2025.9.18-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5f96fa342b6f54dcba928dd452e8d8cb9f0d63e711d1721cd765bb9f73bb048d"}, + {file = "regex-2025.9.18-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0f0d676522d68c207828dcd01fb6f214f63f238c283d9f01d85fc664c7c85b56"}, + {file = "regex-2025.9.18-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:40532bff8a1a0621e7903ae57fce88feb2e8a9a9116d341701302c9302aef06e"}, + {file = "regex-2025.9.18-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:039f11b618ce8d71a1c364fdee37da1012f5a3e79b1b2819a9f389cd82fd6282"}, + {file = "regex-2025.9.18-cp312-cp312-win32.whl", hash = "sha256:e1dd06f981eb226edf87c55d523131ade7285137fbde837c34dc9d1bf309f459"}, + {file = "regex-2025.9.18-cp312-cp312-win_amd64.whl", hash = "sha256:3d86b5247bf25fa3715e385aa9ff272c307e0636ce0c9595f64568b41f0a9c77"}, + {file = "regex-2025.9.18-cp312-cp312-win_arm64.whl", hash = "sha256:032720248cbeeae6444c269b78cb15664458b7bb9ed02401d3da59fe4d68c3a5"}, + {file = "regex-2025.9.18-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2a40f929cd907c7e8ac7566ac76225a77701a6221bca937bdb70d56cb61f57b2"}, + {file = "regex-2025.9.18-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c90471671c2cdf914e58b6af62420ea9ecd06d1554d7474d50133ff26ae88feb"}, + {file = "regex-2025.9.18-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1a351aff9e07a2dabb5022ead6380cff17a4f10e4feb15f9100ee56c4d6d06af"}, + {file = "regex-2025.9.18-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc4b8e9d16e20ddfe16430c23468a8707ccad3365b06d4536142e71823f3ca29"}, + {file = "regex-2025.9.18-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4b8cdbddf2db1c5e80338ba2daa3cfa3dec73a46fff2a7dda087c8efbf12d62f"}, + {file = "regex-2025.9.18-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a276937d9d75085b2c91fb48244349c6954f05ee97bba0963ce24a9d915b8b68"}, + {file = "regex-2025.9.18-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92a8e375ccdc1256401c90e9dc02b8642894443d549ff5e25e36d7cf8a80c783"}, + {file = "regex-2025.9.18-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0dc6893b1f502d73037cf807a321cdc9be29ef3d6219f7970f842475873712ac"}, + {file = "regex-2025.9.18-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a61e85bfc63d232ac14b015af1261f826260c8deb19401c0597dbb87a864361e"}, + {file = "regex-2025.9.18-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:1ef86a9ebc53f379d921fb9a7e42b92059ad3ee800fcd9e0fe6181090e9f6c23"}, + {file = "regex-2025.9.18-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d3bc882119764ba3a119fbf2bd4f1b47bc56c1da5d42df4ed54ae1e8e66fdf8f"}, + {file = "regex-2025.9.18-cp313-cp313-win32.whl", hash = "sha256:3810a65675845c3bdfa58c3c7d88624356dd6ee2fc186628295e0969005f928d"}, + {file = "regex-2025.9.18-cp313-cp313-win_amd64.whl", hash = "sha256:16eaf74b3c4180ede88f620f299e474913ab6924d5c4b89b3833bc2345d83b3d"}, + {file = "regex-2025.9.18-cp313-cp313-win_arm64.whl", hash = "sha256:4dc98ba7dd66bd1261927a9f49bd5ee2bcb3660f7962f1ec02617280fc00f5eb"}, + {file = "regex-2025.9.18-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:fe5d50572bc885a0a799410a717c42b1a6b50e2f45872e2b40f4f288f9bce8a2"}, + {file = "regex-2025.9.18-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:1b9d9a2d6cda6621551ca8cf7a06f103adf72831153f3c0d982386110870c4d3"}, + {file = "regex-2025.9.18-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:13202e4c4ac0ef9a317fff817674b293c8f7e8c68d3190377d8d8b749f566e12"}, + {file = "regex-2025.9.18-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874ff523b0fecffb090f80ae53dc93538f8db954c8bb5505f05b7787ab3402a0"}, + {file = "regex-2025.9.18-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d13ab0490128f2bb45d596f754148cd750411afc97e813e4b3a61cf278a23bb6"}, + {file = "regex-2025.9.18-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:05440bc172bc4b4b37fb9667e796597419404dbba62e171e1f826d7d2a9ebcef"}, + {file = "regex-2025.9.18-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5514b8e4031fdfaa3d27e92c75719cbe7f379e28cacd939807289bce76d0e35a"}, + {file = "regex-2025.9.18-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:65d3c38c39efce73e0d9dc019697b39903ba25b1ad45ebbd730d2cf32741f40d"}, + {file = "regex-2025.9.18-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:ae77e447ebc144d5a26d50055c6ddba1d6ad4a865a560ec7200b8b06bc529368"}, + {file = "regex-2025.9.18-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:e3ef8cf53dc8df49d7e28a356cf824e3623764e9833348b655cfed4524ab8a90"}, + {file = "regex-2025.9.18-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:9feb29817df349c976da9a0debf775c5c33fc1c8ad7b9f025825da99374770b7"}, + {file = "regex-2025.9.18-cp313-cp313t-win32.whl", hash = "sha256:168be0d2f9b9d13076940b1ed774f98595b4e3c7fc54584bba81b3cc4181742e"}, + {file = "regex-2025.9.18-cp313-cp313t-win_amd64.whl", hash = "sha256:d59ecf3bb549e491c8104fea7313f3563c7b048e01287db0a90485734a70a730"}, + {file = "regex-2025.9.18-cp313-cp313t-win_arm64.whl", hash = "sha256:dbef80defe9fb21310948a2595420b36c6d641d9bea4c991175829b2cc4bc06a"}, + {file = "regex-2025.9.18-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:c6db75b51acf277997f3adcd0ad89045d856190d13359f15ab5dda21581d9129"}, + {file = "regex-2025.9.18-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8f9698b6f6895d6db810e0bda5364f9ceb9e5b11328700a90cae573574f61eea"}, + {file = "regex-2025.9.18-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:29cd86aa7cb13a37d0f0d7c21d8d949fe402ffa0ea697e635afedd97ab4b69f1"}, + {file = "regex-2025.9.18-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c9f285a071ee55cd9583ba24dde006e53e17780bb309baa8e4289cd472bcc47"}, + {file = "regex-2025.9.18-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5adf266f730431e3be9021d3e5b8d5ee65e563fec2883ea8093944d21863b379"}, + {file = "regex-2025.9.18-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1137cabc0f38807de79e28d3f6e3e3f2cc8cfb26bead754d02e6d1de5f679203"}, + {file = "regex-2025.9.18-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7cc9e5525cada99699ca9223cce2d52e88c52a3d2a0e842bd53de5497c604164"}, + {file = "regex-2025.9.18-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bbb9246568f72dce29bcd433517c2be22c7791784b223a810225af3b50d1aafb"}, + {file = "regex-2025.9.18-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:6a52219a93dd3d92c675383efff6ae18c982e2d7651c792b1e6d121055808743"}, + {file = "regex-2025.9.18-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:ae9b3840c5bd456780e3ddf2f737ab55a79b790f6409182012718a35c6d43282"}, + {file = "regex-2025.9.18-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d488c236ac497c46a5ac2005a952c1a0e22a07be9f10c3e735bc7d1209a34773"}, + {file = "regex-2025.9.18-cp314-cp314-win32.whl", hash = "sha256:0c3506682ea19beefe627a38872d8da65cc01ffa25ed3f2e422dffa1474f0788"}, + {file = "regex-2025.9.18-cp314-cp314-win_amd64.whl", hash = "sha256:57929d0f92bebb2d1a83af372cd0ffba2263f13f376e19b1e4fa32aec4efddc3"}, + {file = "regex-2025.9.18-cp314-cp314-win_arm64.whl", hash = "sha256:6a4b44df31d34fa51aa5c995d3aa3c999cec4d69b9bd414a8be51984d859f06d"}, + {file = "regex-2025.9.18-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:b176326bcd544b5e9b17d6943f807697c0cb7351f6cfb45bf5637c95ff7e6306"}, + {file = "regex-2025.9.18-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:0ffd9e230b826b15b369391bec167baed57c7ce39efc35835448618860995946"}, + {file = "regex-2025.9.18-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec46332c41add73f2b57e2f5b642f991f6b15e50e9f86285e08ffe3a512ac39f"}, + {file = "regex-2025.9.18-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b80fa342ed1ea095168a3f116637bd1030d39c9ff38dc04e54ef7c521e01fc95"}, + {file = "regex-2025.9.18-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f4d97071c0ba40f0cf2a93ed76e660654c399a0a04ab7d85472239460f3da84b"}, + {file = "regex-2025.9.18-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0ac936537ad87cef9e0e66c5144484206c1354224ee811ab1519a32373e411f3"}, + {file = "regex-2025.9.18-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dec57f96d4def58c422d212d414efe28218d58537b5445cf0c33afb1b4768571"}, + {file = "regex-2025.9.18-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:48317233294648bf7cd068857f248e3a57222259a5304d32c7552e2284a1b2ad"}, + {file = "regex-2025.9.18-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:274687e62ea3cf54846a9b25fc48a04459de50af30a7bd0b61a9e38015983494"}, + {file = "regex-2025.9.18-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a78722c86a3e7e6aadf9579e3b0ad78d955f2d1f1a8ca4f67d7ca258e8719d4b"}, + {file = "regex-2025.9.18-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:06104cd203cdef3ade989a1c45b6215bf42f8b9dd705ecc220c173233f7cba41"}, + {file = "regex-2025.9.18-cp314-cp314t-win32.whl", hash = "sha256:2e1eddc06eeaffd249c0adb6fafc19e2118e6308c60df9db27919e96b5656096"}, + {file = "regex-2025.9.18-cp314-cp314t-win_amd64.whl", hash = "sha256:8620d247fb8c0683ade51217b459cb4a1081c0405a3072235ba43a40d355c09a"}, + {file = "regex-2025.9.18-cp314-cp314t-win_arm64.whl", hash = "sha256:b7531a8ef61de2c647cdf68b3229b071e46ec326b3138b2180acb4275f470b01"}, + {file = "regex-2025.9.18-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3dbcfcaa18e9480669030d07371713c10b4f1a41f791ffa5cb1a99f24e777f40"}, + {file = "regex-2025.9.18-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1e85f73ef7095f0380208269055ae20524bfde3f27c5384126ddccf20382a638"}, + {file = "regex-2025.9.18-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9098e29b3ea4ffffeade423f6779665e2a4f8db64e699c0ed737ef0db6ba7b12"}, + {file = "regex-2025.9.18-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90b6b7a2d0f45b7ecaaee1aec6b362184d6596ba2092dd583ffba1b78dd0231c"}, + {file = "regex-2025.9.18-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c81b892af4a38286101502eae7aec69f7cd749a893d9987a92776954f3943408"}, + {file = "regex-2025.9.18-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3b524d010973f2e1929aeb635418d468d869a5f77b52084d9f74c272189c251d"}, + {file = "regex-2025.9.18-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b498437c026a3d5d0be0020023ff76d70ae4d77118e92f6f26c9d0423452446"}, + {file = "regex-2025.9.18-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0716e4d6e58853d83f6563f3cf25c281ff46cf7107e5f11879e32cb0b59797d9"}, + {file = "regex-2025.9.18-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:065b6956749379d41db2625f880b637d4acc14c0a4de0d25d609a62850e96d36"}, + {file = "regex-2025.9.18-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d4a691494439287c08ddb9b5793da605ee80299dd31e95fa3f323fac3c33d9d4"}, + {file = "regex-2025.9.18-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ef8d10cc0989565bcbe45fb4439f044594d5c2b8919d3d229ea2c4238f1d55b0"}, + {file = "regex-2025.9.18-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4baeb1b16735ac969a7eeecc216f1f8b7caf60431f38a2671ae601f716a32d25"}, + {file = "regex-2025.9.18-cp39-cp39-win32.whl", hash = "sha256:8e5f41ad24a1e0b5dfcf4c4e5d9f5bd54c895feb5708dd0c1d0d35693b24d478"}, + {file = "regex-2025.9.18-cp39-cp39-win_amd64.whl", hash = "sha256:50e8290707f2fb8e314ab3831e594da71e062f1d623b05266f8cfe4db4949afd"}, + {file = "regex-2025.9.18-cp39-cp39-win_arm64.whl", hash = "sha256:039a9d7195fd88c943d7c777d4941e8ef736731947becce773c31a1009cb3c35"}, + {file = "regex-2025.9.18.tar.gz", hash = "sha256:c5ba23274c61c6fef447ba6a39333297d0c247f53059dba0bca415cac511edc4"}, +] + [[package]] name = "requests" version = "2.32.5" @@ -930,7 +1078,7 @@ type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.deve name = "six" version = "1.17.0" description = "Python 2 and 3 compatibility utilities" -optional = true +optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] files = [ @@ -1009,7 +1157,7 @@ files = [ name = "tzdata" version = "2025.2" description = "Provider of IANA time zone data" -optional = true +optional = false python-versions = ">=2" groups = ["main"] files = [ @@ -1017,6 +1165,24 @@ files = [ {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, ] +[[package]] +name = "tzlocal" +version = "5.3.1" +description = "tzinfo object for the local timezone" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d"}, + {file = "tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd"}, +] + +[package.dependencies] +tzdata = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +devenv = ["check-manifest", "pytest (>=4.3)", "pytest-cov", "pytest-mock (>=3.3)", "zest.releaser"] + [[package]] name = "urllib3" version = "2.5.0" @@ -1077,4 +1243,4 @@ ruyaml = ">=0.91.0" [metadata] lock-version = "2.1" python-versions = "^3.9" -content-hash = "350cf05513620283f7727ac4146751814bd61f42f08579c7048e17def8679a91" +content-hash = "5c107fedb6a29dd4a0296b2d6d9bd451fcda356054d200900420e4df52c2fdc7" diff --git a/pyproject.toml b/pyproject.toml index 26083f7..3a78e26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,8 @@ click = "^8.1.8" hecdss = { version = ">=0.1.24", optional = true } # Via https://github.com/HydrologicEngineeringCenter/hec-python-library/blob/main/hec/shared.py#L9-10 cwms-python = { version = ">=0.8.0", optional = true} pyyaml = "^6.0.2" +python-dateutil = "2.9.0" +dateparser = "1.2.0" [tool.poetry.group.dev.dependencies] black = "^24.2.0" From c745f251b1242e330677736340c5ccd51b4bbb7a Mon Sep 17 00:00:00 2001 From: Charles Graham SWT Date: Tue, 23 Sep 2025 09:40:09 -0500 Subject: [PATCH 11/25] Restructure from single file --- cwmscli/reporting/__init__.py | 660 +------------------------------- cwmscli/reporting/config.py | 148 +++++++ cwmscli/reporting/core.py | 283 ++++++++++++++ cwmscli/reporting/models.py | 49 +++ cwmscli/reporting/utils/date.py | 84 ++++ 5 files changed, 585 insertions(+), 639 deletions(-) create mode 100644 cwmscli/reporting/config.py create mode 100644 cwmscli/reporting/core.py create mode 100644 cwmscli/reporting/models.py create mode 100644 cwmscli/reporting/utils/date.py diff --git a/cwmscli/reporting/__init__.py b/cwmscli/reporting/__init__.py index 37e6eb1..44ba157 100644 --- a/cwmscli/reporting/__init__.py +++ b/cwmscli/reporting/__init__.py @@ -1,447 +1,24 @@ from __future__ import annotations -import math import os import traceback -from dataclasses import dataclass, field -from datetime import datetime, timedelta, timezone +from datetime import timezone from typing import Any, Dict, List, Optional -from zoneinfo import ZoneInfo import click import cwms -import pandas as pd -import yaml +from cwmscli.reporting.config import Config +from cwmscli.reporting.core import build_report_table +from cwmscli.reporting.utils.date import parse_range from cwmscli.utils.deps import requires -@dataclass -class ProjectSpec: - location_id: str - href: Optional[str] = None - office: Optional[str] = None - - -@dataclass -class ReportSpec: - district: str - name: str - logo_left: Optional[str] = None - logo_right: Optional[str] = None - - -@dataclass -class HeaderCellSpec: - text: str - colspan: int = 1 - rowspan: int = 1 - align: Optional[str] = None # "left"|"center"|"right" - classes: Optional[str] = None - - -@dataclass -class TableHeaderSpec: - project: HeaderCellSpec = field( - default_factory=lambda: HeaderCellSpec(text="Project", rowspan=1) - ) - rows: List[List[HeaderCellSpec]] = field(default_factory=list) - - -@dataclass -class ColumnSpec: - title: str - key: str - tsid: Optional[str] = None - level: Optional[str] = None - unit: Optional[str] = None - precision: Optional[int] = None - office: Optional[str] = None - location_id: Optional[str] = None - href: Optional[str] = None - missing: Optional[str] = None - undefined: Optional[str] = None - target_time: Optional[str] = None - - -@dataclass -class Config: - office: str - cda_api_root: Optional[str] = None - report: ReportSpec | Dict[str, Any] | None = None - projects: List[ProjectSpec] = field(default_factory=list) - columns: List[ColumnSpec] = field(default_factory=list) - header: Optional[TableHeaderSpec] = None - begin: Optional[str] = None - end: Optional[str] = None - - target_time: Optional[str] = None - time_epsilon_minutes: int = 5 - - default_unit: str = "EN" - missing: str = "----" - undefined: str = "--NA--" - time_zone: Optional[str] = None - - @staticmethod - def from_yaml(path: str) -> "Config": - with open(path, "r", encoding="utf-8") as f: - raw = yaml.safe_load(f) or {} - - office = ( - raw.get("office") - or os.getenv("OFFICE") - or os.getenv("CWMS_OFFICE") - or "SWT" - ) - - report_block = raw.get("report") or {} - report = ReportSpec( - district=report_block.get("district", office), - name=report_block.get("name", "Daily Report"), - logo_left=report_block.get("logo_left"), - logo_right=report_block.get("logo_right"), - ) - - cols: List[ColumnSpec] = [] - for i, c in enumerate(raw.get("columns", [])): - cols.append( - ColumnSpec( - title=c.get("title") or c.get("name") or f"Col{i+1}", - key=c.get("key") or c.get("title") or f"c{i+1}", - tsid=c.get("tsid"), - level=c.get("level"), - unit=c.get("unit"), - precision=c.get("precision"), - office=c.get("office"), - location_id=c.get("location_id"), - href=c.get("href"), - missing=c.get("missing"), - undefined=c.get("undefined"), - target_time=c.get("target_time"), - ) - ) - - projects_raw = raw.get("projects", []) - projects: List[ProjectSpec] = [] - for p in projects_raw: - if isinstance(p, str): - projects.append(ProjectSpec(location_id=p)) - elif isinstance(p, dict): - projects.append( - ProjectSpec( - location_id=p.get("location_id") - or p.get("name") - or p.get("id"), - href=p.get("href"), - office=p.get("office"), - ) - ) - else: - raise click.BadParameter(f"Invalid project entry: {p!r}") - # Validate the columns and header spec - header = _parse_header_spec(raw.get("header")) - if header and header.rows: - # compute leaf-count in the final header row - leaf_count = sum(max(1, c.colspan) for c in header.rows[-1]) - if leaf_count != len(cols): - click.echo( - f"[reporting] Warning: header leaf-count ({leaf_count}) != number of data columns ({len(cols)}).", - err=True, - ) - return Config( - office=office, - cda_api_root=raw.get("cda_api_root") or os.getenv("CDA_API_ROOT"), - report=report, - projects=projects, - columns=cols, - begin=raw.get("begin"), - end=raw.get("end"), - target_time=raw.get("target_time"), - time_epsilon_minutes=int(raw.get("time_epsilon_minutes") or 5), - default_unit=raw.get("default_unit") or "EN", - missing=raw.get("missing") or "----", - undefined=raw.get("undefined") or "--NA--", - time_zone=raw.get("time_zone"), - header=header, - ) - - -def _parse_header_spec(raw: Optional[Dict[str, Any]]) -> Optional["TableHeaderSpec"]: - if not raw: - return None - - def to_cell(d: Dict[str, Any]) -> HeaderCellSpec: - return HeaderCellSpec( - text=str(d.get("text", "")), - colspan=int(d.get("colspan", 1) or 1), - rowspan=int(d.get("rowspan", 1) or 1), - align=d.get("align"), - classes=d.get("classes"), - ) - - proj_raw = raw.get("project", {}) or {} - project = to_cell( - { - "text": proj_raw.get("text", "Project"), - "rowspan": proj_raw.get("rowspan", 1), - "align": proj_raw.get("align"), - "classes": proj_raw.get("classes"), - } - ) - rows_raw = raw.get("rows", []) or [] - rows = [] - for r in rows_raw: - row_cells = [to_cell(c) for c in (r or [])] - rows.append(row_cells) - return TableHeaderSpec(project=project, rows=rows) - - -def _parse_target_like( - s: Optional[str], default_tz: Optional[str] -) -> Optional[datetime]: - """ - Accepts: - - ISO (with/without tz): '2025-09-17T08:00:00-05:00', '2025-09-17T13:00Z' - - 'HHMM YYYY-MM-DD [TZ]', 'HHMM MM/DD/YYYY [TZ]', 'HH:MM MM/DD/YYYY [TZ]' - - '0800 09/17/2025 America/Chicago' - - 'today 08:00 [TZ]' or 'yesterday 08:00 [TZ]' (optional) - Returns timezone-aware UTC datetime. - """ - if not s: - return None - s = " ".join(str(s).split()) - - lower = s.lower() - if lower.startswith(("today", "yesterday")): - parts = s.split() - base = ( - datetime.now(ZoneInfo(default_tz)) - if default_tz - else datetime.now(timezone.utc) - ) - if parts[0].lower() == "yesterday": - base = base - timedelta(days=1) - - hhmm = parts[1] if len(parts) > 1 else "00:00" - tz = parts[2] if len(parts) > 2 else default_tz - if ":" in hhmm: - hh, mm = hhmm.split(":") - else: - hh, mm = hhmm[:2], hhmm[2:] - naive = base.replace(hour=int(hh), minute=int(mm), second=0, microsecond=0) - aware = ( - naive - if naive.tzinfo - else ( - naive.replace(tzinfo=ZoneInfo(tz)) - if tz - else naive.replace(tzinfo=timezone.utc) - ) - ) - return aware.astimezone(timezone.utc) - - try: - dt = datetime.fromisoformat(s) - if dt.tzinfo is None: - dt = dt.replace(tzinfo=ZoneInfo(default_tz) if default_tz else timezone.utc) - return dt.astimezone(timezone.utc) - except Exception: - pass - - try: - parts = s.split() - tz = None - if len(parts) == 3: - time_s, date_s, tz = parts - elif len(parts) == 2: - time_s, date_s = parts - else: - raise ValueError() - - if ":" in time_s: - hh, mm = time_s.split(":") - else: - hh, mm = time_s[:2], time_s[2:] - - if "/" in date_s: - mon, day, yr = date_s.split("/") - yr = int(yr) - mon = int(mon) - day = int(day) - elif "-" in date_s: - yr, mon, day = date_s.split("-") - yr = int(yr) - mon = int(mon) - day = int(day) - else: - raise ValueError() - - tzinfo = ( - ZoneInfo(tz) - if tz - else (ZoneInfo(default_tz) if default_tz else timezone.utc) - ) - local = datetime(yr, mon, day, int(hh), int(mm), tzinfo=tzinfo) - return local.astimezone(timezone.utc) - except Exception: - raise click.BadParameter(f"Invalid target_time: {s}") - - -def _parse_time_or_relative(s: Optional[str]) -> Optional[datetime]: - """ - Accepts ISO strings (with or without tz) or relative like "24h", "3d", "90m". - Returns timezone-aware UTC datetimes (or None). - """ - if not s: - return None - s = str(s).strip() - - if s.endswith(("h", "m", "d")) and s[:-1].isdigit(): - amount = int(s[:-1]) - unit = s[-1] - now = datetime.now(timezone.utc) - if unit == "h": - return now - timedelta(hours=amount) - if unit == "m": - return now - timedelta(minutes=amount) - if unit == "d": - return now - timedelta(days=amount) - - try: - dt = datetime.fromisoformat(s) - except ValueError: - raise click.BadParameter(f"Invalid datetime: {s}") - - if dt.tzinfo is None: - dt = dt.replace(tzinfo=timezone.utc) - return dt.astimezone(timezone.utc) - - -def _format_number(x: Any, precision: Optional[int]) -> str: - if x is None or (isinstance(x, float) and (math.isnan(x) or math.isinf(x))): - return "-" - try: - if precision is None: - return f"{x}" - fmt = f"{{:.{precision}f}}" - return fmt.format(float(x)) - except Exception: - return f"{x}" - - -def _expand_tsid(tsid_template: str, project: str) -> str: - """ - If tsid_template contains '{project}', substitute it. - Otherwise return as-is (full TSIDs remain unchanged). - """ - if "{project}" in tsid_template: - return tsid_template.format(project=project) - return tsid_template - - -def _fetch_multi_df( - tsids: List[str], - office: str, - unit: str, - begin: Optional[datetime], - end: Optional[datetime], -) -> pd.DataFrame: - """ - Wrapper around cwms.get_multi_timeseries_df, always returns a melted frame: - columns: ['date-time','name','value','quality-code'] (depending on cwms-python version) - """ - df = cwms.get_multi_timeseries_df( - ts_ids=tsids, - office_id=office, - unit=unit, - begin=begin, - end=end, - melted=True, - ) - - if "date-time" in df.columns: - df["date-time"] = pd.to_datetime(df["date-time"], utc=True, errors="coerce") - return df - - -def _window_for_target(dt: datetime, minutes: int) -> tuple[datetime, datetime]: - eps = max(1, int(minutes)) - return (dt - timedelta(minutes=eps), dt + timedelta(minutes=eps)) - - -def _expand_template(s: Optional[str], **kwargs) -> Optional[str]: - if not s: - return None - try: - return s.format(**kwargs) - except Exception: - return s - - -def _fetch_levels_dict( - level_ids: List[str], - begin: str, - end: str, - office: str, - unit: str, -) -> Dict[str, float | None]: - """ - Return {level_id: value or None}. - We assume cwms-python supports a get_level-like call; fall back to levels endpoint if needed. - """ - out: Dict[str, float | None] = {} - for lvl in level_ids: - try: - - val = cwms.get_level_as_timeseries( - begin=datetime.fromisoformat(begin), - end=datetime.fromisoformat(end), - location_level_id=lvl, - office_id=office, - unit=unit, - ) - out[lvl] = val.json.get("values")[-1][1] if val.json.get("values") else None - except Exception as err: - print( - f"[reporting] Warning: could not fetch level '{lvl}': {err}", - traceback.format_exc(), - ) - out[lvl] = None - return out - - -def _format_value( - x: Any, - precision: Optional[int], - missing: str, - undefined: str, -) -> str: - - if x is None: - return missing - try: - xf = float(x) - if math.isnan(xf) or math.isinf(xf): - return undefined - if precision is None: - return f"{xf}" - return f"{xf:.{precision}f}" - except Exception: - - return f"{x}" - - def _render_template( template_dir: Optional[str], template_name: str | None, context: Dict[str, Any], ) -> str: - """ - Try user-specified template directory first; if not provided or missing, - fall back to package templates (if you ship any). For now, we only support - user-supplied templates or very simple built-in fallback. - """ import jinja2 loaders: List[jinja2.BaseLoader] = [] @@ -473,201 +50,6 @@ def _render_template( click.echo(traceback.format_exc()) -def build_report_table( - config: Config, begin: Optional[datetime], end: Optional[datetime] -) -> Dict[str, Any]: - rows: List[str] = [p.location_id for p in config.projects] - if not rows: - raise click.UsageError("No 'projects' configured in YAML.") - - proj_by_id: Dict[str, ProjectSpec] = {p.location_id: p for p in config.projects} - - col_defs: List[Dict[str, Any]] = [] - for c in config.columns: - if not (c.tsid or c.level): - raise click.BadParameter(f"Column '{c.title}' must have 'tsid' or 'level'.") - col_defs.append( - { - "title": c.title, - "key": c.key, - "precision": c.precision, - "unit": c.unit or config.default_unit, - "office": c.office or config.office, - "tsid_template": c.tsid, - "level_template": c.level, - "href_template": c.href, - "missing": c.missing or config.missing, - "undefined": c.undefined or config.undefined, - "target_time": c.target_time or config.target_time, - } - ) - - base_end = end or datetime.now(timezone.utc) - - ts_groups: Dict[tuple, List[str]] = {} - backref_ts: Dict[tuple, List[tuple]] = {} - - lvl_groups: Dict[tuple, List[str]] = {} - backref_lvl: Dict[tuple, List[tuple]] = {} - - col_time_windows: Dict[str, tuple[datetime, datetime] | None] = {} - for c in col_defs: - tt = c.get("target_time") - if tt: - dt = _parse_target_like(tt, config.time_zone) - col_time_windows[c["key"]] = _window_for_target( - dt, config.time_epsilon_minutes - ) - else: - col_time_windows[c["key"]] = None - - for proj_id in rows: - for c in col_defs: - office = c["office"] - unit = c["unit"] - key = c["key"] - - if c["tsid_template"]: - tsid = _expand_template(c["tsid_template"], project=proj_id) - win = col_time_windows[key] - b, e = win if win else (begin, end) - - if b is None or e is None: - e = e or base_end - b = b or (e - timedelta(hours=24)) - gk = (office, unit, b, e) - ts_groups.setdefault(gk, []) - if tsid not in ts_groups[gk]: - ts_groups[gk].append(tsid) - backref_ts.setdefault((office, unit, b, e, tsid), []).append( - (proj_id, key) - ) - - elif c["level_template"]: - lvl = _expand_template(c["level_template"], project=proj_id) - gk = (office, unit) - lvl_groups.setdefault(gk, []) - if lvl not in lvl_groups[gk]: - lvl_groups[gk].append(lvl) - backref_lvl.setdefault((office, unit, lvl), []).append((proj_id, key)) - - last_ts_value: Dict[tuple, float | None] = {} - for (office, unit, b, e), tsids in ts_groups.items(): - if not tsids: - continue - df = _fetch_multi_df(tsids, office, unit, b, e) - - name_col = ( - "ts_id" - if "ts_id" in df.columns - else ("name" if "name" in df.columns else None) - ) - time_col = ( - "date-time" - if "date-time" in df.columns - else ("date_time" if "date_time" in df.columns else None) - ) - if name_col and time_col: - df = df.dropna(subset=[time_col]) - df[time_col] = pd.to_datetime(df[time_col], utc=True, errors="coerce") - df = df.sort_values([name_col, time_col]) - last = df.groupby(name_col).tail(1) - for _, row in last.iterrows(): - last_ts_value[(office, unit, b, e, str(row[name_col]))] = row.get( - "value", None - ) - else: - - for ts in tsids: - last_ts_value[(office, unit, b, e, ts)] = None - - last_lvl_value: Dict[tuple, float | None] = {} - for (office, unit), lvls in lvl_groups.items(): - if not lvls: - continue - vals = _fetch_levels_dict( - lvls, - begin=config.begin, - end=config.end, - office=office, - unit=unit, - ) - for lvl in lvls: - last_lvl_value[(office, unit, lvl)] = vals.get(lvl) - - proj_locations: Dict[str, Dict[str, Any]] = {} - for proj_id in rows: - proj = proj_by_id[proj_id] - proj_office = proj.office or config.office - try: - loc = cwms.get_location(office_id=proj_office, location_id=proj_id) - loc_json = getattr(loc, "json", None) or loc - if isinstance(loc_json, dict): - loc_json = {**loc_json} - if proj.href: - loc_json["href"] = proj.href - else: - loc_json = {"name": proj_id, "href": proj.href} - except Exception: - loc_json = {"name": proj_id, "href": proj.href} - proj_locations[proj_id] = loc_json - - table: Dict[str, Dict[str, Any]] = {proj_id: {} for proj_id in rows} - - for (office, unit, b, e, tsid), pairs in backref_ts.items(): - raw = last_ts_value.get((office, unit, b, e, tsid)) - - for proj_id, col_key in pairs: - c = next((x for x in col_defs if x["key"] == col_key), None) - val_text = _format_value( - raw, - precision=c.get("precision") if c else None, - missing=(c.get("missing") or config.missing), - undefined=(c.get("undefined") or config.undefined), - ) - href = _expand_template( - c.get("href_template"), - project=proj_id, - office=office, - tsid=tsid, - level=None, - ) - table[proj_id][col_key] = ( - {"text": val_text, "href": href} if href else {"text": val_text} - ) - - for (office, unit, lvl), pairs in backref_lvl.items(): - raw = last_lvl_value.get((office, unit, lvl)) - for proj_id, col_key in pairs: - c = next((x for x in col_defs if x["key"] == col_key), None) - val_text = _format_value( - raw, - precision=c.get("precision") if c else None, - missing=(c.get("missing") or config.missing), - undefined=(c.get("undefined") or config.undefined), - ) - href = _expand_template( - c.get("href_template"), - project=proj_id, - office=office, - tsid=None, - level=lvl, - ) - table[proj_id][col_key] = ( - {"text": val_text, "href": href} if href else {"text": val_text} - ) - - for proj_id in rows: - table[proj_id]["location"] = proj_locations.get(proj_id, {"name": proj_id}) - - return { - "columns": col_defs, - "rows": rows, - "data": table, - "base_end": base_end, - } - - @click.command( name="reporting", help="Render a CWMS timeseries report to HTML using a YAML config and Jinja2.", @@ -694,11 +76,6 @@ def build_report_table( default=None, help="Template filename to render (relative to --template-dir). Default: report.html.j2", ) -@click.option( - "--begin", - help='Override begin time (ISO or relative like "24h"). If omitted, uses YAML or defaults.', -) -@click.option("--end", help="Override end time (ISO). If omitted, uses YAML or now.") @click.option( "--out", "-o", @@ -716,22 +93,27 @@ def build_report_table( "desc": "Templating for pre/post-processing", }, ) -def reporting_cli(config_path, template_dir, template_name, begin, end, out_path): - +def reporting_cli(config_path, template_dir, template_name, out_path): cfg = Config.from_yaml(config_path) - cfg_begin = ( - _parse_time_or_relative(begin) if begin else _parse_time_or_relative(cfg.begin) - ) - cfg_end = _parse_time_or_relative(end) if end else _parse_time_or_relative(cfg.end) - if cfg_end is None: - cfg_end = datetime.now(timezone.utc) - if cfg_begin is None: - cfg_begin = cfg_end - timedelta(hours=24) + + # Resolve begin/end from YAML ONLY (no CLI overrides) + tz = cfg.time_zone or "UTC" + if not cfg.begin or not cfg.end: + raise click.ClickException( + "Config must specify 'begin' and 'end' strings (ISO, ISO with % placeholders, or natural language)." + ) + + begin_dt, end_dt = parse_range(cfg.begin, cfg.end, tz) + # guard + if end_dt <= begin_dt: + raise click.ClickException( + f"'end' ({end_dt.isoformat()}) must be after 'begin' ({begin_dt.isoformat()})" + ) cwms.init_session(api_root=cfg.cda_api_root) - table_ctx = build_report_table(cfg, cfg_begin, cfg_end) + table_ctx = build_report_table(cfg, begin_dt, end_dt) - base_date = table_ctx.get("base_end", cfg_end).astimezone(timezone.utc) + base_date = table_ctx.get("base_end", end_dt).astimezone(timezone.utc) context = { "office": cfg.office, "report": dataclasses_asdict(cfg.report), diff --git a/cwmscli/reporting/config.py b/cwmscli/reporting/config.py new file mode 100644 index 0000000..3a70679 --- /dev/null +++ b/cwmscli/reporting/config.py @@ -0,0 +1,148 @@ +import os +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +import click +import yaml + +from cwmscli.reporting.models import ( + ColumnSpec, + HeaderCellSpec, + ProjectSpec, + ReportSpec, + TableHeaderSpec, +) + + +def _parse_header_spec(raw: Optional[Dict[str, Any]]) -> Optional["TableHeaderSpec"]: + if not raw: + return None + + def to_cell(d: Dict[str, Any]) -> HeaderCellSpec: + return HeaderCellSpec( + text=str(d.get("text", "")), + colspan=int(d.get("colspan", 1) or 1), + rowspan=int(d.get("rowspan", 1) or 1), + align=d.get("align"), + classes=d.get("classes"), + ) + + proj_raw = raw.get("project", {}) or {} + project = to_cell( + { + "text": proj_raw.get("text", "Project"), + "rowspan": proj_raw.get("rowspan", 1), + "align": proj_raw.get("align"), + "classes": proj_raw.get("classes"), + } + ) + rows_raw = raw.get("rows", []) or [] + rows = [] + for r in rows_raw: + row_cells = [to_cell(c) for c in (r or [])] + rows.append(row_cells) + return TableHeaderSpec(project=project, rows=rows) + + +@dataclass +class Config: + office: str + cda_api_root: Optional[str] = None + report: ReportSpec | Dict[str, Any] | None = None + projects: List[ProjectSpec] = field(default_factory=list) + columns: List[ColumnSpec] = field(default_factory=list) + header: Optional[TableHeaderSpec] = None + + # REQUIRED in YAML now + begin: Optional[str] = None + end: Optional[str] = None + + # target_time removed (global) + time_epsilon_minutes: int = 5 # kept, in case you want windowing later + + default_unit: str = "EN" + missing: str = "----" + undefined: str = "--NA--" + time_zone: Optional[str] = None # e.g., "America/Chicago" + + @staticmethod + def from_yaml(path: str) -> "Config": + with open(path, "r", encoding="utf-8") as f: + raw = yaml.safe_load(f) or {} + + office = ( + raw.get("office") + or os.getenv("OFFICE") + or os.getenv("CWMS_OFFICE") + or "SWT" + ) + + report_block = raw.get("report") or {} + report = ReportSpec( + district=report_block.get("district", office), + name=report_block.get("name", "Daily Report"), + logo_left=report_block.get("logo_left"), + logo_right=report_block.get("logo_right"), + ) + + cols: List[ColumnSpec] = [] + for i, c in enumerate(raw.get("columns", [])): + cols.append( + ColumnSpec( + title=c.get("title") or c.get("name") or f"Col{i+1}", + key=c.get("key") or c.get("title") or f"c{i+1}", + tsid=c.get("tsid"), + level=c.get("level"), + unit=c.get("unit"), + precision=c.get("precision"), + office=c.get("office"), + location_id=c.get("location_id"), + href=c.get("href"), + missing=c.get("missing"), + undefined=c.get("undefined"), + ) + ) + + projects_raw = raw.get("projects", []) + projects: List[ProjectSpec] = [] + for p in projects_raw: + if isinstance(p, str): + projects.append(ProjectSpec(location_id=p)) + elif isinstance(p, dict): + projects.append( + ProjectSpec( + location_id=p.get("location_id") + or p.get("name") + or p.get("id"), + href=p.get("href"), + office=p.get("office"), + ) + ) + else: + raise click.BadParameter(f"Invalid project entry: {p!r}") + + header = _parse_header_spec(raw.get("header")) + if header and header.rows: + # compute leaf-count in the final header row + leaf_count = sum(max(1, c.colspan) for c in header.rows[-1]) + if leaf_count != len(cols): + click.echo( + f"[reporting] Warning: header leaf-count ({leaf_count}) != number of data columns ({len(cols)}).", + err=True, + ) + + return Config( + office=office, + cda_api_root=raw.get("cda_api_root") or os.getenv("CDA_API_ROOT"), + report=report, + projects=projects, + columns=cols, + begin=raw.get("begin"), + end=raw.get("end"), + time_epsilon_minutes=int(raw.get("time_epsilon_minutes") or 5), + default_unit=raw.get("default_unit") or "EN", + missing=raw.get("missing") or "----", + undefined=raw.get("undefined") or "--NA--", + time_zone=raw.get("time_zone"), + header=header, + ) diff --git a/cwmscli/reporting/core.py b/cwmscli/reporting/core.py new file mode 100644 index 0000000..3548904 --- /dev/null +++ b/cwmscli/reporting/core.py @@ -0,0 +1,283 @@ +# ========================= +# Core +# ========================= + + +import math +import traceback +from datetime import datetime +from typing import Any, Dict, List, Optional + +import click +import cwms +import pandas as pd + +from cwmscli.reporting.config import Config +from cwmscli.reporting.models import ProjectSpec + + +def _expand_template(s: Optional[str], **kwargs) -> Optional[str]: + if not s: + return None + try: + return s.format(**kwargs) + except Exception: + return s + + +def _fetch_multi_df( + tsids: List[str], + office: str, + unit: str, + begin: Optional[datetime], + end: Optional[datetime], +) -> pd.DataFrame: + """ + Wrapper around cwms.get_multi_timeseries_df, always returns a melted frame: + columns: ['date-time','name','value','quality-code'] (depending on cwms-python version) + """ + df = cwms.get_multi_timeseries_df( + ts_ids=tsids, + office_id=office, + unit=unit, + begin=begin, + end=end, + melted=True, + ) + if "date-time" in df.columns: + df["date-time"] = pd.to_datetime(df["date-time"], utc=True, errors="coerce") + return df + + +def _fetch_levels_dict( + level_ids: List[str], + begin: datetime, + end: datetime, + office: str, + unit: str, +) -> Dict[str, float | None]: + """ + Return {level_id: value or None}. + """ + out: Dict[str, float | None] = {} + for lvl in level_ids: + try: + val = cwms.get_level_as_timeseries( + begin=begin, + end=end, + location_level_id=lvl, + office_id=office, + unit=unit, + ) + js = getattr(val, "json", None) or {} + if callable(js): + js = val.json() + values = (js or {}).get("values", []) + out[lvl] = values[-1][1] if values else None + except Exception as err: + print( + f"[reporting] Warning: could not fetch level '{lvl}': {err}", + traceback.format_exc(), + ) + out[lvl] = None + return out + + +def _format_value( + x: Any, + precision: Optional[int], + missing: str, + undefined: str, +) -> str: + if x is None: + return missing + try: + xf = float(x) + if math.isnan(xf) or math.isinf(xf): + return undefined + if precision is None: + return f"{xf}" + return f"{xf:.{precision}f}" + except Exception: + return f"{x}" + + +def build_report_table( + config: Config, begin: datetime, end: datetime +) -> Dict[str, Any]: + rows: List[str] = [p.location_id for p in config.projects] + if not rows: + raise click.UsageError("No 'projects' configured in YAML.") + + proj_by_id: Dict[str, ProjectSpec] = {p.location_id: p for p in config.projects} + + col_defs: List[Dict[str, Any]] = [] + for c in config.columns: + if not (c.tsid or c.level): + raise click.BadParameter(f"Column '{c.title}' must have 'tsid' or 'level'.") + col_defs.append( + { + "title": c.title, + "key": c.key, + "precision": c.precision, + "unit": c.unit or config.default_unit, + "office": c.office or config.office, + "tsid_template": c.tsid, + "level_template": c.level, + "href_template": c.href, + "missing": c.missing or config.missing, + "undefined": c.undefined or config.undefined, + } + ) + + base_end = end or datetime.now(timezone.utc) + + # Group ts requests by (office, unit, begin, end) + ts_groups: Dict[tuple, List[str]] = {} + backref_ts: Dict[tuple, List[tuple]] = {} + + # Group level requests by (office, unit) + lvl_groups: Dict[tuple, List[str]] = {} + backref_lvl: Dict[tuple, List[tuple]] = {} + + for proj_id in rows: + for c in col_defs: + office = c["office"] + unit = c["unit"] + key = c["key"] + + if c["tsid_template"]: + tsid = _expand_template(c["tsid_template"], project=proj_id) + gk = (office, unit, begin, end) + ts_groups.setdefault(gk, []) + if tsid not in ts_groups[gk]: + ts_groups[gk].append(tsid) + backref_ts.setdefault((office, unit, begin, end, tsid), []).append( + (proj_id, key) + ) + + elif c["level_template"]: + lvl = _expand_template(c["level_template"], project=proj_id) + gk = (office, unit) + lvl_groups.setdefault(gk, []) + if lvl not in lvl_groups[gk]: + lvl_groups[gk].append(lvl) + backref_lvl.setdefault((office, unit, lvl), []).append((proj_id, key)) + + # Fetch latest TS values within window + last_ts_value: Dict[tuple, float | None] = {} + for (office, unit, b, e), tsids in ts_groups.items(): + if not tsids: + continue + df = _fetch_multi_df(tsids, office, unit, b, e) + + name_col = ( + "ts_id" + if "ts_id" in df.columns + else ("name" if "name" in df.columns else None) + ) + time_col = ( + "date-time" + if "date-time" in df.columns + else ("date_time" if "date_time" in df.columns else None) + ) + if name_col and time_col: + df = df.dropna(subset=[time_col]) + df[time_col] = pd.to_datetime(df[time_col], utc=True, errors="coerce") + df = df.sort_values([name_col, time_col]) + last = df.groupby(name_col).tail(1) + for _, row in last.iterrows(): + last_ts_value[(office, unit, b, e, str(row[name_col]))] = row.get( + "value", None + ) + else: + for ts in tsids: + last_ts_value[(office, unit, b, e, ts)] = None + + # Fetch latest Level values + last_lvl_value: Dict[tuple, float | None] = {} + for (office, unit), lvls in lvl_groups.items(): + if not lvls: + continue + vals = _fetch_levels_dict( + lvls, + begin=begin, + end=end, + office=office, + unit=unit, + ) + for lvl in lvls: + last_lvl_value[(office, unit, lvl)] = vals.get(lvl) + + # Project location info + proj_locations: Dict[str, Dict[str, Any]] = {} + for proj_id in rows: + proj = proj_by_id[proj_id] + proj_office = proj.office or config.office + try: + loc = cwms.get_location(office_id=proj_office, location_id=proj_id) + loc_json = getattr(loc, "json", None) or loc + if isinstance(loc_json, dict): + loc_json = {**loc_json} + if proj.href: + loc_json["href"] = proj.href + else: + loc_json = {"name": proj_id, "href": proj.href} + except Exception: + loc_json = {"name": proj_id, "href": proj.href} + proj_locations[proj_id] = loc_json + + # Build table payload + table: Dict[str, Dict[str, Any]] = {proj_id: {} for proj_id in rows} + + for (office, unit, b, e, tsid), pairs in backref_ts.items(): + raw = last_ts_value.get((office, unit, b, e, tsid)) + for proj_id, col_key in pairs: + c = next((x for x in col_defs if x["key"] == col_key), None) + val_text = _format_value( + raw, + precision=c.get("precision") if c else None, + missing=(c.get("missing") or config.missing), + undefined=(c.get("undefined") or config.undefined), + ) + href = _expand_template( + c.get("href_template"), + project=proj_id, + office=office, + tsid=tsid, + level=None, + ) + table[proj_id][col_key] = ( + {"text": val_text, "href": href} if href else {"text": val_text} + ) + + for (office, unit, lvl), pairs in backref_lvl.items(): + raw = last_lvl_value.get((office, unit, lvl)) + for proj_id, col_key in pairs: + c = next((x for x in col_defs if x["key"] == col_key), None) + val_text = _format_value( + raw, + precision=c.get("precision") if c else None, + missing=(c.get("missing") or config.missing), + undefined=(c.get("undefined") or config.undefined), + ) + href = _expand_template( + c.get("href_template"), + project=proj_id, + office=office, + tsid=None, + level=lvl, + ) + table[proj_id][col_key] = ( + {"text": val_text, "href": href} if href else {"text": val_text} + ) + + for proj_id in rows: + table[proj_id]["location"] = proj_locations.get(proj_id, {"name": proj_id}) + + return { + "columns": col_defs, + "rows": rows, + "data": table, + "base_end": base_end, + } diff --git a/cwmscli/reporting/models.py b/cwmscli/reporting/models.py new file mode 100644 index 0000000..281be3b --- /dev/null +++ b/cwmscli/reporting/models.py @@ -0,0 +1,49 @@ +from dataclasses import dataclass, field +from typing import List, Optional + + +@dataclass +class ProjectSpec: + location_id: str + href: Optional[str] = None + office: Optional[str] = None + + +@dataclass +class ReportSpec: + district: str + name: str + logo_left: Optional[str] = None + logo_right: Optional[str] = None + + +@dataclass +class HeaderCellSpec: + text: str + colspan: int = 1 + rowspan: int = 1 + align: Optional[str] = None + classes: Optional[str] = None + + +@dataclass +class TableHeaderSpec: + project: HeaderCellSpec = field( + default_factory=lambda: HeaderCellSpec(text="Project", rowspan=1) + ) + rows: List[List[HeaderCellSpec]] = field(default_factory=list) + + +@dataclass +class ColumnSpec: + title: str + key: str + tsid: Optional[str] = None + level: Optional[str] = None + unit: Optional[str] = None + precision: Optional[int] = None + office: Optional[str] = None + location_id: Optional[str] = None + href: Optional[str] = None + missing: Optional[str] = None + undefined: Optional[str] = None diff --git a/cwmscli/reporting/utils/date.py b/cwmscli/reporting/utils/date.py new file mode 100644 index 0000000..74e5e71 --- /dev/null +++ b/cwmscli/reporting/utils/date.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from datetime import datetime +from zoneinfo import ZoneInfo + + +def parse_when(expr: str, tz: str = "GMT", *, _now: datetime | None = None) -> datetime: + """ + Parse a flexible datetime expression: + - ISO 8601 (e.g. 2025-09-22T08:00[:SS][Z|±HH:MM]) + - ISO with strftime placeholders (e.g. "%Y-%m-01T08:00:00") + - Natural language (e.g. "2 years ago September 1 08:00", "yesterday 08:00") + Returns a timezone-aware datetime in the provided tz. + """ + s = (expr or "").strip() + if not s: + raise ValueError("empty datetime expression") + + tzinfo = ZoneInfo(tz) + now = _now or datetime.now(tzinfo) + + # Expand strftime placeholders first if any + if "%" in s: + s = now.strftime(s) + + # Try strict ISO first + try: + iso = s.replace("Z", "+00:00") if s.endswith("Z") else s + dt = datetime.fromisoformat(iso) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=tzinfo) + else: + dt = dt.astimezone(tzinfo) + return dt + except Exception: + pass + + # Give options to the parsers + # - dateutil.parser: https://dateutil.readthedocs.io/en/stable/parser.html + # - dateparser: https://dateparser.readthedocs.io + try: + from dateutil import parser as du_parser + + dt = du_parser.parse(s) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=tzinfo) + else: + dt = dt.astimezone(tzinfo) + return dt + except Exception: + pass + + try: + from dateparser import parse as dp_parse + + dt = dp_parse( + s, + settings={ + "RETURN_AS_TIMEZONE_AWARE": True, + "TIMEZONE": tz, + "PREFER_DAY_OF_MONTH": "first", + }, + ) + if dt: + # convert to expected tz if not already + if dt.tzinfo is None: + dt = dt.replace(tzinfo=tzinfo) + else: + dt = dt.astimezone(tzinfo) + return dt + except Exception: + pass + + raise ValueError(f"Could not parse datetime expression: {expr!r}") + + +def parse_range(begin_expr: str, end_expr: str, tz: str = "America/Chicago"): + begin = parse_when(begin_expr, tz) + end = parse_when(end_expr, tz) + if end <= begin: + raise ValueError( + f"end ({end.isoformat()}) must be after begin ({begin.isoformat()})" + ) + return begin, end From 39fe299a42ab723320427e3cb145d0236c97b442 Mon Sep 17 00:00:00 2001 From: Charles Graham SWT Date: Tue, 23 Sep 2025 09:42:25 -0500 Subject: [PATCH 12/25] Create reporting docs dir --- cwmscli/reporting/{ => docs}/reporting_plan.excalidraw | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cwmscli/reporting/{ => docs}/reporting_plan.excalidraw (100%) diff --git a/cwmscli/reporting/reporting_plan.excalidraw b/cwmscli/reporting/docs/reporting_plan.excalidraw similarity index 100% rename from cwmscli/reporting/reporting_plan.excalidraw rename to cwmscli/reporting/docs/reporting_plan.excalidraw From e56dc6a58d99199e455152481b36ff501a2fd36f Mon Sep 17 00:00:00 2001 From: Charles Graham SWT Date: Tue, 23 Sep 2025 15:35:32 -0500 Subject: [PATCH 13/25] Add per column item begin/end times --- cwmscli/reporting/__init__.py | 24 +++-- cwmscli/reporting/config.py | 9 +- cwmscli/reporting/configs/daily.yaml | 5 +- cwmscli/reporting/core.py | 139 +++++++++++++++------------ cwmscli/reporting/models.py | 2 + 5 files changed, 98 insertions(+), 81 deletions(-) diff --git a/cwmscli/reporting/__init__.py b/cwmscli/reporting/__init__.py index 44ba157..b00a8f5 100644 --- a/cwmscli/reporting/__init__.py +++ b/cwmscli/reporting/__init__.py @@ -10,7 +10,7 @@ from cwmscli.reporting.config import Config from cwmscli.reporting.core import build_report_table -from cwmscli.reporting.utils.date import parse_range +from cwmscli.reporting.utils.date import parse_when from cwmscli.utils.deps import requires @@ -96,16 +96,14 @@ def _render_template( def reporting_cli(config_path, template_dir, template_name, out_path): cfg = Config.from_yaml(config_path) - # Resolve begin/end from YAML ONLY (no CLI overrides) tz = cfg.time_zone or "UTC" - if not cfg.begin or not cfg.end: - raise click.ClickException( - "Config must specify 'begin' and 'end' strings (ISO, ISO with % placeholders, or natural language)." - ) - begin_dt, end_dt = parse_range(cfg.begin, cfg.end, tz) - # guard - if end_dt <= begin_dt: + # Global window: optional + begin_dt: Optional[datetime] = parse_when(cfg.begin, tz) if cfg.begin else None + end_dt: Optional[datetime] = parse_when(cfg.end, tz) if cfg.end else None + + # If both provided, sanity check ordering + if begin_dt and end_dt and end_dt < begin_dt: raise click.ClickException( f"'end' ({end_dt.isoformat()}) must be after 'begin' ({begin_dt.isoformat()})" ) @@ -113,7 +111,9 @@ def reporting_cli(config_path, template_dir, template_name, out_path): cwms.init_session(api_root=cfg.cda_api_root) table_ctx = build_report_table(cfg, begin_dt, end_dt) - base_date = table_ctx.get("base_end", end_dt).astimezone(timezone.utc) + base_date = table_ctx.get( + "base_end", end_dt or datetime.now(timezone.utc) + ).astimezone(timezone.utc) context = { "office": cfg.office, "report": dataclasses_asdict(cfg.report), @@ -130,6 +130,10 @@ def reporting_cli(config_path, template_dir, template_name, out_path): def dataclasses_asdict(obj): + # Custom dataclass to dict, recursive + # Guarantees we end up with a structure made of only "safe" Python types: + # dicts, lists, tuples, numbers, strings, None. + # Helper for Jinja2 or JSON data structures if obj is None: return None if hasattr(obj, "__dataclass_fields__"): diff --git a/cwmscli/reporting/config.py b/cwmscli/reporting/config.py index 3a70679..6b6ca2b 100644 --- a/cwmscli/reporting/config.py +++ b/cwmscli/reporting/config.py @@ -52,18 +52,13 @@ class Config: projects: List[ProjectSpec] = field(default_factory=list) columns: List[ColumnSpec] = field(default_factory=list) header: Optional[TableHeaderSpec] = None - - # REQUIRED in YAML now begin: Optional[str] = None end: Optional[str] = None - - # target_time removed (global) - time_epsilon_minutes: int = 5 # kept, in case you want windowing later - + time_epsilon_minutes: int = 5 default_unit: str = "EN" missing: str = "----" undefined: str = "--NA--" - time_zone: Optional[str] = None # e.g., "America/Chicago" + time_zone: Optional[str] = None # i.e. "America/Chicago" @staticmethod def from_yaml(path: str) -> "Config": diff --git a/cwmscli/reporting/configs/daily.yaml b/cwmscli/reporting/configs/daily.yaml index eebff1f..38ab483 100644 --- a/cwmscli/reporting/configs/daily.yaml +++ b/cwmscli/reporting/configs/daily.yaml @@ -176,7 +176,8 @@ columns: precision: 2 tsid: "{project}.Elev.Inst.1Hour.0.Ccp-Rev" # override the global target time for THIS column (optional): - target_time: "2025-09-22T08:00:00-05:00" + begin: "today 0800" + end: "today 0800" # per-column render strings (optional): missing: "--" undefined: "~~~~.~~" @@ -187,6 +188,8 @@ columns: unit: "EN" precision: 2 tsid: "{project}.Elev.Inst.1Hour.0.Ccp-Rev" + begin: "yesterday 0800 America/Chicago" + end: "yesterday 0800 America/Chicago" # override the global target time for THIS column (optional): target_time: "2025-09-21T08:00:00-05:00" # per-column render strings (optional): diff --git a/cwmscli/reporting/core.py b/cwmscli/reporting/core.py index 3548904..984f149 100644 --- a/cwmscli/reporting/core.py +++ b/cwmscli/reporting/core.py @@ -1,11 +1,6 @@ -# ========================= -# Core -# ========================= - - import math import traceback -from datetime import datetime +from datetime import datetime, timezone from typing import Any, Dict, List, Optional import click @@ -14,6 +9,7 @@ from cwmscli.reporting.config import Config from cwmscli.reporting.models import ProjectSpec +from cwmscli.reporting.utils.date import parse_when def _expand_template(s: Optional[str], **kwargs) -> Optional[str]: @@ -32,10 +28,6 @@ def _fetch_multi_df( begin: Optional[datetime], end: Optional[datetime], ) -> pd.DataFrame: - """ - Wrapper around cwms.get_multi_timeseries_df, always returns a melted frame: - columns: ['date-time','name','value','quality-code'] (depending on cwms-python version) - """ df = cwms.get_multi_timeseries_df( ts_ids=tsids, office_id=office, @@ -51,14 +43,11 @@ def _fetch_multi_df( def _fetch_levels_dict( level_ids: List[str], - begin: datetime, - end: datetime, + begin: Optional[datetime], + end: Optional[datetime], office: str, unit: str, ) -> Dict[str, float | None]: - """ - Return {level_id: value or None}. - """ out: Dict[str, float | None] = {} for lvl in level_ids: try: @@ -103,13 +92,14 @@ def _format_value( def build_report_table( - config: Config, begin: datetime, end: datetime + config: Config, begin: Optional[datetime], end: Optional[datetime] ) -> Dict[str, Any]: rows: List[str] = [p.location_id for p in config.projects] if not rows: raise click.UsageError("No 'projects' configured in YAML.") proj_by_id: Dict[str, ProjectSpec] = {p.location_id: p for p in config.projects} + tz = config.time_zone or "UTC" col_defs: List[Dict[str, Any]] = [] for c in config.columns: @@ -127,49 +117,72 @@ def build_report_table( "href_template": c.href, "missing": c.missing or config.missing, "undefined": c.undefined or config.undefined, + "begin_expr": c.begin, + "end_expr": c.end, } ) - base_end = end or datetime.now(timezone.utc) + candidate_ends: List[datetime] = [] + if end: + candidate_ends.append(end) + + def effective_range( + bexpr: Optional[str], eexpr: Optional[str] + ) -> tuple[Optional[datetime], Optional[datetime]]: + b_eff = parse_when(bexpr, tz) if bexpr else begin + e_eff = parse_when(eexpr, tz) if eexpr else end + return b_eff, e_eff - # Group ts requests by (office, unit, begin, end) ts_groups: Dict[tuple, List[str]] = {} backref_ts: Dict[tuple, List[tuple]] = {} - # Group level requests by (office, unit) lvl_groups: Dict[tuple, List[str]] = {} backref_lvl: Dict[tuple, List[tuple]] = {} + effective_windows: Dict[tuple, tuple[Optional[datetime], Optional[datetime]]] = {} + for proj_id in rows: for c in col_defs: office = c["office"] unit = c["unit"] key = c["key"] + b_eff, e_eff = effective_range(c.get("begin_expr"), c.get("end_expr")) + if e_eff: + candidate_ends.append(e_eff) + effective_windows[(proj_id, key)] = (b_eff, e_eff) + if c["tsid_template"]: tsid = _expand_template(c["tsid_template"], project=proj_id) - gk = (office, unit, begin, end) + gk = (office, unit, b_eff, e_eff) ts_groups.setdefault(gk, []) if tsid not in ts_groups[gk]: ts_groups[gk].append(tsid) - backref_ts.setdefault((office, unit, begin, end, tsid), []).append( + backref_ts.setdefault((office, unit, b_eff, e_eff, tsid), []).append( (proj_id, key) ) elif c["level_template"]: lvl = _expand_template(c["level_template"], project=proj_id) - gk = (office, unit) + gk = (office, unit, b_eff, e_eff) lvl_groups.setdefault(gk, []) if lvl not in lvl_groups[gk]: lvl_groups[gk].append(lvl) - backref_lvl.setdefault((office, unit, lvl), []).append((proj_id, key)) + backref_lvl.setdefault((office, unit, b_eff, e_eff, lvl), []).append( + (proj_id, key) + ) + + base_end = ( + candidate_ends + and max(dt for dt in candidate_ends if dt is not None) + or datetime.now(timezone.utc) + ) - # Fetch latest TS values within window last_ts_value: Dict[tuple, float | None] = {} - for (office, unit, b, e), tsids in ts_groups.items(): + for (office, unit, b_eff, e_eff), tsids in ts_groups.items(): if not tsids: continue - df = _fetch_multi_df(tsids, office, unit, b, e) + df = _fetch_multi_df(tsids, office, unit, b_eff, e_eff) name_col = ( "ts_id" @@ -181,57 +194,38 @@ def build_report_table( if "date-time" in df.columns else ("date_time" if "date_time" in df.columns else None) ) + if name_col and time_col: df = df.dropna(subset=[time_col]) df[time_col] = pd.to_datetime(df[time_col], utc=True, errors="coerce") df = df.sort_values([name_col, time_col]) last = df.groupby(name_col).tail(1) for _, row in last.iterrows(): - last_ts_value[(office, unit, b, e, str(row[name_col]))] = row.get( - "value", None + last_ts_value[(office, unit, b_eff, e_eff, str(row[name_col]))] = ( + row.get("value", None) ) else: for ts in tsids: - last_ts_value[(office, unit, b, e, ts)] = None + last_ts_value[(office, unit, b_eff, e_eff, ts)] = None - # Fetch latest Level values last_lvl_value: Dict[tuple, float | None] = {} - for (office, unit), lvls in lvl_groups.items(): + for (office, unit, b_eff, e_eff), lvls in lvl_groups.items(): if not lvls: continue vals = _fetch_levels_dict( lvls, - begin=begin, - end=end, + begin=b_eff, + end=e_eff, office=office, unit=unit, ) for lvl in lvls: - last_lvl_value[(office, unit, lvl)] = vals.get(lvl) - - # Project location info - proj_locations: Dict[str, Dict[str, Any]] = {} - for proj_id in rows: - proj = proj_by_id[proj_id] - proj_office = proj.office or config.office - try: - loc = cwms.get_location(office_id=proj_office, location_id=proj_id) - loc_json = getattr(loc, "json", None) or loc - if isinstance(loc_json, dict): - loc_json = {**loc_json} - if proj.href: - loc_json["href"] = proj.href - else: - loc_json = {"name": proj_id, "href": proj.href} - except Exception: - loc_json = {"name": proj_id, "href": proj.href} - proj_locations[proj_id] = loc_json + last_lvl_value[(office, unit, b_eff, e_eff, lvl)] = vals.get(lvl) - # Build table payload table: Dict[str, Dict[str, Any]] = {proj_id: {} for proj_id in rows} - for (office, unit, b, e, tsid), pairs in backref_ts.items(): - raw = last_ts_value.get((office, unit, b, e, tsid)) + for (office, unit, b_eff, e_eff, tsid), pairs in backref_ts.items(): + raw = last_ts_value.get((office, unit, b_eff, e_eff, tsid)) for proj_id, col_key in pairs: c = next((x for x in col_defs if x["key"] == col_key), None) val_text = _format_value( @@ -247,12 +241,13 @@ def build_report_table( tsid=tsid, level=None, ) - table[proj_id][col_key] = ( - {"text": val_text, "href": href} if href else {"text": val_text} - ) + table[proj_id][col_key] = { + "text": val_text, + **({"href": href} if href else {}), + } - for (office, unit, lvl), pairs in backref_lvl.items(): - raw = last_lvl_value.get((office, unit, lvl)) + for (office, unit, b_eff, e_eff, lvl), pairs in backref_lvl.items(): + raw = last_lvl_value.get((office, unit, b_eff, e_eff, lvl)) for proj_id, col_key in pairs: c = next((x for x in col_defs if x["key"] == col_key), None) val_text = _format_value( @@ -268,9 +263,27 @@ def build_report_table( tsid=None, level=lvl, ) - table[proj_id][col_key] = ( - {"text": val_text, "href": href} if href else {"text": val_text} - ) + table[proj_id][col_key] = { + "text": val_text, + **({"href": href} if href else {}), + } + + proj_locations: Dict[str, Dict[str, Any]] = {} + for proj_id in rows: + proj = proj_by_id[proj_id] + proj_office = proj.office or config.office + try: + loc = cwms.get_location(office_id=proj_office, location_id=proj_id) + loc_json = getattr(loc, "json", None) or loc + if isinstance(loc_json, dict): + loc_json = {**loc_json} + if proj.href: + loc_json["href"] = proj.href + else: + loc_json = {"name": proj_id, "href": proj.href} + except Exception: + loc_json = {"name": proj_id, "href": proj.href} + proj_locations[proj_id] = loc_json for proj_id in rows: table[proj_id]["location"] = proj_locations.get(proj_id, {"name": proj_id}) diff --git a/cwmscli/reporting/models.py b/cwmscli/reporting/models.py index 281be3b..702836c 100644 --- a/cwmscli/reporting/models.py +++ b/cwmscli/reporting/models.py @@ -47,3 +47,5 @@ class ColumnSpec: href: Optional[str] = None missing: Optional[str] = None undefined: Optional[str] = None + begin: Optional[str] = None + end: Optional[str] = None From 7a24adeb94cc9fc690d943ab9334002b7ce0363c Mon Sep 17 00:00:00 2001 From: Charles Graham SWT Date: Tue, 30 Sep 2025 09:35:15 -0500 Subject: [PATCH 14/25] Remove time delta idea --- cwmscli/reporting/configs/daily.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/cwmscli/reporting/configs/daily.yaml b/cwmscli/reporting/configs/daily.yaml index 38ab483..32a4de8 100644 --- a/cwmscli/reporting/configs/daily.yaml +++ b/cwmscli/reporting/configs/daily.yaml @@ -7,7 +7,6 @@ begin: "2025-09-16T00:00:00-05:00" end: "2025-09-17T00:00:00-05:00" # Or pin to a single instant, globally: target_time: "0800 09/17/2025 America/Chicago" -time_epsilon_minutes: 10 # +/- window around target_time for TS picks; default 5 default_unit: "EN" missing: "----" # used when no value at all was found From 84f30f2db44d260093e8803fa18fd7a1583dcb9b Mon Sep 17 00:00:00 2001 From: Charles Graham Date: Fri, 19 Sep 2025 12:10:29 -0500 Subject: [PATCH 15/25] 20 setup read the docs documentation (#31) I tried to set this up so that it could also use sphinx-click to automatically show the commands from the click arguments we are using. Part of that and using sphinx was warning me and would not build if I tried to run python `3.9`. @Enovotny if you are working on 3.12 or 3.13 for the T7, is it reasonable to mark 3.11 as the minimum for sphinx/click to work in the RTD gen? ( was not able to generate with the minimum in the toml set to 3.9 or 3.10 ) I also setup a CI/CD action that should make it so a PR fails if the docs fail to build. Action should only run if one of those directories change(?) --- .github/workflows/docs.yml | 25 ++ .readthedocs.yaml | 16 ++ README.md | 6 +- cwmscli/commands/commands_cwms.py | 15 +- cwmscli/utils/deps.py | 4 +- docs/Makefile | 20 ++ docs/cli.rst | 6 + docs/cli/blob.rst | 8 + docs/conf.py | 37 +++ docs/index.rst | 5 + docs/make.bat | 35 +++ docs/requirements.txt | 3 + poetry.lock | 381 +++++++++++++++++++++++++++++- pyproject.toml | 8 +- 14 files changed, 552 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/docs.yml create mode 100644 .readthedocs.yaml create mode 100644 docs/Makefile create mode 100644 docs/cli.rst create mode 100644 docs/cli/blob.rst create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/make.bat create mode 100644 docs/requirements.txt diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..ba5fa82 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,25 @@ +# Confirm the docs build works on PRs touching docs or code +name: ReadTheDocs/Sphinx Validation +on: + pull_request: + paths: ["docs/**", "cwmscli/**", "pyproject.toml"] + push: + branches: [main] + paths: ["docs/**", "cwmscli/**", "pyproject.toml"] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install deps + run: | + python -m pip install -U pip + pip install -r docs/requirements.txt + pip install . + - name: Sphinx build (treat warnings as errors) + run: sphinx-build -nW -b html docs docs/_build/html + - name: Link check (optional) + run: sphinx-build -b linkcheck docs docs/_build/linkcheck diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..51d46cb --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,16 @@ +version: 2 + +build: + os: ubuntu-24.04 + tools: + python: "3.12" + +sphinx: + configuration: docs/conf.py + +python: + install: + - method: pip + path: . + - requirements: docs/requirements.txt +formats: [html, pdf] diff --git a/README.md b/README.md index 7078850..ec92580 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,17 @@ # cwms-cli + command line utilities used for Corps Water Management Systems (CWMS) processes +[![Docs](https://readthedocs.org/projects/cwms-cli/badge/?version=latest)](https://cwms-cli.readthedocs.io/en/latest/) + ## Install + ```sh pip install git+https://github.com/HydrologicEngineeringCenter/cwms-cli.git@main ``` ## Command line implementation + ```sh cwms-cli --help ``` - diff --git a/cwmscli/commands/commands_cwms.py b/cwmscli/commands/commands_cwms.py index 0c6a524..cec79da 100644 --- a/cwmscli/commands/commands_cwms.py +++ b/cwmscli/commands/commands_cwms.py @@ -100,7 +100,16 @@ def csv2cwms_cmd(**kwargs): # ================================================================================ # BLOB # ================================================================================ -@click.group("blob", help="Manage CWMS Blobs (upload, download, delete, update, list)") +@click.group( + "blob", + help="Manage CWMS Blobs (upload, download, delete, update, list)", + epilog=""" + * Store a PDF/image as a CWMS blob with optional description + * Download a blob by id to your local filesystem + * Update a blob's name/description + * Bulk list blobs for an office +""", +) @requires(reqs.cwms) def blob_group(): pass @@ -190,7 +199,9 @@ def update_cmd(**kwargs): # ================================================================================ @blob_group.command("list", help="List blobs with optional filters and sorting") # TODO: Add link to regex docs when new CWMS-DATA site is deployed to PROD -@click.option("--blob-id-like", help="LIKE filter for blob ID (e.g., '*PNG').") +@click.option( + "--blob-id-like", help="LIKE filter for blob ID (e.g., ``*PNG``)." +) # Escape the wildcard/asterisk for RTD generation with double backticks @click.option( "--columns", multiple=True, diff --git a/cwmscli/utils/deps.py b/cwmscli/utils/deps.py index c3e9e88..05bc02f 100644 --- a/cwmscli/utils/deps.py +++ b/cwmscli/utils/deps.py @@ -32,7 +32,7 @@ def requires(*requirements): "package": "cwms-python", "version": "0.8.0", "desc": "CWMS REST API Python client", - "link": "https://github.com/USACE/cwms-python" + "link": "https://github.com/hydrologicengineeringcenter/cwms-python" }, { "module": "requests", @@ -40,8 +40,6 @@ def requires(*requirements): "desc": "Required for HTTP API access" } ) - def my_command(): - ... """ def decorator(func): diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/cli.rst b/docs/cli.rst new file mode 100644 index 0000000..b1e0844 --- /dev/null +++ b/docs/cli.rst @@ -0,0 +1,6 @@ +CLI reference +============= + +.. click:: cwmscli.__main__:cli + :prog: cwms-cli + :nested: full diff --git a/docs/cli/blob.rst b/docs/cli/blob.rst new file mode 100644 index 0000000..9ef0817 --- /dev/null +++ b/docs/cli/blob.rst @@ -0,0 +1,8 @@ +Blob commands +============= + +Overview, examples, etc… + +.. click:: cwmscli.commands.commands_cwms:blob_group + :prog: cwms-cli blob + :nested: full \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..59c5a25 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,37 @@ +import importlib.metadata as ilmd +import os +import sys + +# Make cwms-cli importable for autodoc/sphinx-click +sys.path.insert(0, os.path.abspath("..")) + +project = "cwms-cli" + +# Get the installed package version without shadowing Sphinx's "version" +try: + pkg_version = ilmd.version("cwms-cli") +except ilmd.PackageNotFoundError: + pkg_version = "0.0.0" + +release = pkg_version +version = pkg_version + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinx.ext.intersphinx", + "sphinx_click", +] + +autosummary_generate = True +autodoc_typehints = "description" + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), +} + +html_theme = "sphinx_rtd_theme" + +# autodoc_mock_imports = ["cwms", "pandas", "requests"] diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..b1f199a --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,5 @@ +.. toctree:: + :maxdepth: 2 + + cli + cli/blob diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..954237b --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..1299c01 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +sphinx +sphinx-rtd-theme +sphinx-click \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 2ba922a..bb06c19 100644 --- a/poetry.lock +++ b/poetry.lock @@ -38,8 +38,6 @@ mypy-extensions = ">=0.4.3" packaging = ">=22.0" pathspec = ">=0.9.0" platformdirs = ">=2" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] @@ -51,7 +49,7 @@ uvloop = ["uvloop (>=0.15.2)"] name = "certifi" version = "2025.8.3" description = "Python package for providing Mozilla's CA Bundle." -optional = true +optional = false python-versions = ">=3.7" groups = ["main"] files = [ @@ -75,7 +73,7 @@ files = [ name = "charset-normalizer" version = "3.4.3" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = true +optional = false python-versions = ">=3.7" groups = ["main"] files = [ @@ -187,6 +185,7 @@ files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +markers = {main = "python_version == \"3.11\" and platform_system == \"Windows\" or python_version >= \"3.12\" and platform_system == \"Windows\"", dev = "python_version == \"3.11\" and platform_system == \"Windows\" or python_version >= \"3.12\" and platform_system == \"Windows\"", docs = "python_version == \"3.11\" and platform_system == \"Windows\" or python_version == \"3.11\" and sys_platform == \"win32\" or python_version >= \"3.12\" and platform_system == \"Windows\" or python_version >= \"3.12\" and sys_platform == \"win32\""} [[package]] name = "cwms-python" @@ -252,6 +251,19 @@ files = [ {file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"}, ] +[[package]] +name = "docutils" +version = "0.21.2" +description = "Docutils -- Python Documentation Utilities" +optional = false +python-versions = ">=3.9" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2"}, + {file = "docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f"}, +] + [[package]] name = "filelock" version = "3.19.1" @@ -299,7 +311,7 @@ license = ["ukkonen"] name = "idna" version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" -optional = true +optional = false python-versions = ">=3.6" groups = ["main"] files = [ @@ -310,6 +322,19 @@ files = [ [package.extras] all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] +[[package]] +name = "imagesize" +version = "1.4.1" +description = "Getting image size from png/jpeg/jpeg2000/gif file" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"}, + {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, +] + [[package]] name = "isort" version = "5.13.2" @@ -325,6 +350,25 @@ files = [ [package.extras] colors = ["colorama (>=0.4.6)"] +[[package]] +name = "jinja2" +version = "3.1.6" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, + {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + [[package]] name = "maison" version = "1.4.2" @@ -342,6 +386,78 @@ click = ">=8.0.1,<9.0.0" pydantic = ">=1.10.13,<2.0.0" toml = ">=0.10.2,<0.11.0" +[[package]] +name = "markupsafe" +version = "3.0.2" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.9" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:eaa0a10b7f72326f1372a713e73c3f739b524b3af41feb43e4921cb529f5929a"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:48032821bbdf20f5799ff537c7ac3d1fba0ba032cfc06194faffa8cda8b560ff"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a9d3f5f0901fdec14d8d2f66ef7d035f2157240a433441719ac9a3fba440b13"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88b49a3b9ff31e19998750c38e030fc7bb937398b1f78cfa599aaef92d693144"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cfad01eed2c2e0c01fd0ecd2ef42c492f7f93902e39a42fc9ee1692961443a29"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1225beacc926f536dc82e45f8a4d68502949dc67eea90eab715dea3a21c1b5f0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3169b1eefae027567d1ce6ee7cae382c57fe26e82775f460f0b2778beaad66c0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:eb7972a85c54febfb25b5c4b4f3af4dcc731994c7da0d8a0b4a6eb0640e1d178"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win32.whl", hash = "sha256:8c4e8c3ce11e1f92f6536ff07154f9d49677ebaaafc32db9db4620bc11ed480f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:6e296a513ca3d94054c2c881cc913116e90fd030ad1c656b3869762b754f5f8a"}, + {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, +] + [[package]] name = "mypy" version = "1.17.1" @@ -393,7 +509,6 @@ files = [ [package.dependencies] mypy_extensions = ">=1.0.0" pathspec = ">=0.9.0" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} typing_extensions = ">=4.6.0" [package.extras] @@ -634,7 +749,6 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.22.4", markers = "python_version < \"3.11\""}, {version = ">=1.23.2", markers = "python_version == \"3.11\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] @@ -782,6 +896,22 @@ typing-extensions = ">=4.2.0" dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] +[[package]] +name = "pygments" +version = "2.19.2" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.8" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, + {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + [[package]] name = "python-dateutil" version = "2.9.0" @@ -1001,7 +1131,7 @@ files = [ name = "requests" version = "2.32.5" description = "Python HTTP for Humans." -optional = true +optional = false python-versions = ">=3.9" groups = ["main"] files = [ @@ -1034,6 +1164,23 @@ files = [ [package.dependencies] requests = ">=2.0.1,<3.0.0" +[[package]] +name = "roman-numerals-py" +version = "3.1.0" +description = "Manipulate well-formed Roman numerals" +optional = false +python-versions = ">=3.9" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "roman_numerals_py-3.1.0-py3-none-any.whl", hash = "sha256:9da2ad2fb670bcf24e81070ceb3be72f6c11c440d73bd579fbeca1e9f330954c"}, + {file = "roman_numerals_py-3.1.0.tar.gz", hash = "sha256:be4bf804f083a4ce001b5eb7e3c0862479d10f94c936f6c4e5f250aa5ff5bd2d"}, +] + +[package.extras] +lint = ["mypy (==1.15.0)", "pyright (==1.1.394)", "ruff (==0.9.7)"] +test = ["pytest (>=8)"] + [[package]] name = "ruyaml" version = "0.91.0" @@ -1086,6 +1233,221 @@ files = [ {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] +[[package]] +name = "snowballstemmer" +version = "3.0.1" +description = "This package provides 32 stemmers for 30 languages generated from Snowball algorithms." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "snowballstemmer-3.0.1-py3-none-any.whl", hash = "sha256:6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064"}, + {file = "snowballstemmer-3.0.1.tar.gz", hash = "sha256:6d5eeeec8e9f84d4d56b847692bacf79bc2c8e90c7f80ca4444ff8b6f2e52895"}, +] + +[[package]] +name = "sphinx" +version = "8.2.3" +description = "Python documentation generator" +optional = false +python-versions = ">=3.11" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "sphinx-8.2.3-py3-none-any.whl", hash = "sha256:4405915165f13521d875a8c29c8970800a0141c14cc5416a38feca4ea5d9b9c3"}, + {file = "sphinx-8.2.3.tar.gz", hash = "sha256:398ad29dee7f63a75888314e9424d40f52ce5a6a87ae88e7071e80af296ec348"}, +] + +[package.dependencies] +alabaster = ">=0.7.14" +babel = ">=2.13" +colorama = {version = ">=0.4.6", markers = "sys_platform == \"win32\""} +docutils = ">=0.20,<0.22" +imagesize = ">=1.3" +Jinja2 = ">=3.1" +packaging = ">=23.0" +Pygments = ">=2.17" +requests = ">=2.30.0" +roman-numerals-py = ">=1.0.0" +snowballstemmer = ">=2.2" +sphinxcontrib-applehelp = ">=1.0.7" +sphinxcontrib-devhelp = ">=1.0.6" +sphinxcontrib-htmlhelp = ">=2.0.6" +sphinxcontrib-jsmath = ">=1.0.1" +sphinxcontrib-qthelp = ">=1.0.6" +sphinxcontrib-serializinghtml = ">=1.1.9" + +[package.extras] +docs = ["sphinxcontrib-websupport"] +lint = ["betterproto (==2.0.0b6)", "mypy (==1.15.0)", "pypi-attestations (==0.0.21)", "pyright (==1.1.395)", "pytest (>=8.0)", "ruff (==0.9.9)", "sphinx-lint (>=0.9)", "types-Pillow (==10.2.0.20240822)", "types-Pygments (==2.19.0.20250219)", "types-colorama (==0.4.15.20240311)", "types-defusedxml (==0.7.0.20240218)", "types-docutils (==0.21.0.20241128)", "types-requests (==2.32.0.20241016)", "types-urllib3 (==1.26.25.14)"] +test = ["cython (>=3.0)", "defusedxml (>=0.7.1)", "pytest (>=8.0)", "pytest-xdist[psutil] (>=3.4)", "setuptools (>=70.0)", "typing_extensions (>=4.9)"] + +[[package]] +name = "sphinx-click" +version = "6.1.0" +description = "Sphinx extension that automatically documents click applications" +optional = false +python-versions = ">=3.10" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "sphinx_click-6.1.0-py3-none-any.whl", hash = "sha256:7dbed856c3d0be75a394da444850d5fc7ecc5694534400aa5ed4f4849a8643f9"}, + {file = "sphinx_click-6.1.0.tar.gz", hash = "sha256:c702e0751c1a0b6ad649e4f7faebd0dc09a3cc7ca3b50f959698383772f50eef"}, +] + +[package.dependencies] +click = ">=8.0" +docutils = "*" +sphinx = ">=4.0" + +[package.extras] +docs = ["reno"] +test = ["pytest", "pytest-cov"] + +[[package]] +name = "sphinx-rtd-theme" +version = "3.0.2" +description = "Read the Docs theme for Sphinx" +optional = false +python-versions = ">=3.8" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "sphinx_rtd_theme-3.0.2-py2.py3-none-any.whl", hash = "sha256:422ccc750c3a3a311de4ae327e82affdaf59eb695ba4936538552f3b00f4ee13"}, + {file = "sphinx_rtd_theme-3.0.2.tar.gz", hash = "sha256:b7457bc25dda723b20b086a670b9953c859eab60a2a03ee8eb2bb23e176e5f85"}, +] + +[package.dependencies] +docutils = ">0.18,<0.22" +sphinx = ">=6,<9" +sphinxcontrib-jquery = ">=4,<5" + +[package.extras] +dev = ["bump2version", "transifex-client", "twine", "wheel"] + +[[package]] +name = "sphinxcontrib-applehelp" +version = "2.0.0" +description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" +optional = false +python-versions = ">=3.9" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5"}, + {file = "sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1"}, +] + +[package.extras] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] +standalone = ["Sphinx (>=5)"] +test = ["pytest"] + +[[package]] +name = "sphinxcontrib-devhelp" +version = "2.0.0" +description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp documents" +optional = false +python-versions = ">=3.9" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2"}, + {file = "sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad"}, +] + +[package.extras] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] +standalone = ["Sphinx (>=5)"] +test = ["pytest"] + +[[package]] +name = "sphinxcontrib-htmlhelp" +version = "2.1.0" +description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" +optional = false +python-versions = ">=3.9" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8"}, + {file = "sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9"}, +] + +[package.extras] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] +standalone = ["Sphinx (>=5)"] +test = ["html5lib", "pytest"] + +[[package]] +name = "sphinxcontrib-jquery" +version = "4.1" +description = "Extension to include jQuery on newer Sphinx releases" +optional = false +python-versions = ">=2.7" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "sphinxcontrib-jquery-4.1.tar.gz", hash = "sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a"}, + {file = "sphinxcontrib_jquery-4.1-py2.py3-none-any.whl", hash = "sha256:f936030d7d0147dd026a4f2b5a57343d233f1fc7b363f68b3d4f1cb0993878ae"}, +] + +[package.dependencies] +Sphinx = ">=1.8" + +[[package]] +name = "sphinxcontrib-jsmath" +version = "1.0.1" +description = "A sphinx extension which renders display math in HTML via JavaScript" +optional = false +python-versions = ">=3.5" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"}, + {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, +] + +[package.extras] +test = ["flake8", "mypy", "pytest"] + +[[package]] +name = "sphinxcontrib-qthelp" +version = "2.0.0" +description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp documents" +optional = false +python-versions = ">=3.9" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb"}, + {file = "sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab"}, +] + +[package.extras] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] +standalone = ["Sphinx (>=5)"] +test = ["defusedxml (>=0.7.1)", "pytest"] + +[[package]] +name = "sphinxcontrib-serializinghtml" +version = "2.0.0" +description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)" +optional = false +python-versions = ">=3.9" +groups = ["docs"] +markers = "python_version == \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331"}, + {file = "sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d"}, +] + +[package.extras] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] +standalone = ["Sphinx (>=5)"] +test = ["pytest"] + [[package]] name = "toml" version = "0.10.2" @@ -1187,7 +1549,7 @@ devenv = ["check-manifest", "pytest (>=4.3)", "pytest-cov", "pytest-mock (>=3.3) name = "urllib3" version = "2.5.0" description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = true +optional = false python-versions = ">=3.9" groups = ["main"] files = [ @@ -1217,7 +1579,6 @@ files = [ distlib = ">=0.3.7,<1" filelock = ">=3.12.2,<4" platformdirs = ">=3.9.1,<5" -typing-extensions = {version = ">=4.13.2", markers = "python_version < \"3.11\""} [package.extras] docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] diff --git a/pyproject.toml b/pyproject.toml index 3a78e26..e1f8401 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ keywords = ["USACE", "CWMS"] authors = ["Hydrologic Engineering Center"] [tool.poetry.dependencies] -python = "^3.9" +python = "^3.11" click = "^8.1.8" hecdss = { version = ">=0.1.24", optional = true } # Via https://github.com/HydrologicEngineeringCenter/hec-python-library/blob/main/hec/shared.py#L9-10 cwms-python = { version = ">=0.8.0", optional = true} @@ -33,6 +33,12 @@ pre-commit = "^3.6.2" #pandas-stubs = "^2.2.1.240316" yamlfix = "^1.16.0" + +[tool.poetry.group.docs.dependencies] +sphinx = "^8.2.3" +sphinx-rtd-theme = "^3.0.2" +sphinx-click = "^6.1.0" + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" From b1da09f3dce74b870564a3497c9038bea4b3ebec Mon Sep 17 00:00:00 2001 From: Charles Graham Date: Fri, 19 Sep 2025 15:10:20 -0500 Subject: [PATCH 16/25] 29 blob script needs lazy import for imghdr (#30) - Fix typos/ extra words in issue templates - add blob options to templates - correct import for library needed to determine download image types, only request it be installed if a user is trying to download an image and the extension cannot be determined --- .github/ISSUE_TEMPLATE/bug-report.yml | 4 +-- .github/ISSUE_TEMPLATE/feature-request.yml | 3 +- cwmscli/commands/blob.py | 39 +++++++++++++++++++--- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index d663550..2e0c97a 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -1,14 +1,14 @@ -name: CWMS CLI Script Issues +name: CWMS-CLI Script Issues description: File a bug report CWMS-CLI scripts labels: ["bug"] body: - - type: dropdown id: script attributes: label: CLI Script description: Select the script this pertains to options: + - blob - cwms-cli - csv2cwms - getusgs-measurements diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml index e7a8d4d..783ce91 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.yml +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -1,4 +1,4 @@ -name: CWMS-CLI Script Issue or Feature Request +name: CWMS-CLI Feature Request description: Request a feature related to CWMS-CLI scripts labels: ["enhancement"] body: @@ -8,6 +8,7 @@ body: label: CLI Script description: Select the script this pertains to options: + - blob - cwms-cli - csv2cwms - getusgs-measurements diff --git a/cwmscli/commands/blob.py b/cwmscli/commands/blob.py index 7155b9c..124fbbd 100644 --- a/cwmscli/commands/blob.py +++ b/cwmscli/commands/blob.py @@ -1,5 +1,4 @@ import base64 -import imghdr import json import logging import mimetypes @@ -13,11 +12,42 @@ import requests from cwmscli.utils import get_api_key +from cwmscli.utils.deps import requires # used to rebuild data URL for images DATA_URL_RE = re.compile(r"^data:(?P[^;]+);base64,(?P.+)$", re.I | re.S) +@requires( + { + "module": "imghdr", + "package": "standard-imghdr", + "version": "3.0.0", + "desc": "Package to help detect image types", + "link": "https://docs.python.org/3/library/imghdr.html", + } +) +def _determine_ext(data: bytes | str, write_type: str) -> str: + """ + Attempt to determine the file extension from the data itself. + Requires the imghdr module (lazy import) to inspect the bytes for image types. + If not an image, defaults to .bin + + Args: + data: The binary data or base64 string to inspect. + write_type: The mode in which the data will be written ('wb' for binary, 'w' for text). + + Returns: + The determined file extension, including the leading dot (e.g., '.png', '.jpg'). + """ + import imghdr + + kind = imghdr.what(None, data) + if kind == "jpeg": + kind = "jpg" + return f".{kind}" if kind else ".bin" + + def _save_base64( b64_or_dataurl: str, dest: str, @@ -48,11 +78,10 @@ def _save_base64( ext = mimetypes.guess_extension(media_type.split(";")[0].lower()) or "" if ext == ".jpe": ext = ".jpg" + # last resort, try to determine from the data itself + # requires imghdr to dig into the bytes to determine image type if not ext: - kind = imghdr.what(None, data) - if kind == "jpeg": - kind = "jpg" - ext = f".{kind}" if kind else ".bin" + ext = _determine_ext(data, write_type) dest = base + ext os.makedirs(os.path.dirname(dest) or ".", exist_ok=True) From e0b5160072dd6e661a0e5b7489314ec564777426 Mon Sep 17 00:00:00 2001 From: Charles Graham Date: Fri, 19 Sep 2025 15:32:16 -0500 Subject: [PATCH 17/25] 18 overhaul csv2cwms (#34) - allow custom date formats in the config - Remove duplicate log handler from cwms-cli root - convert projects key to input_files for clarity - provide per file store rule option in the config - fix module name utils to a relative path to assist with import errors --- cwmscli/commands/commands_cwms.py | 15 +-- cwmscli/commands/csv2cwms/README.md | 68 ++++++----- cwmscli/commands/csv2cwms/__main__.py | 108 +++++++++--------- .../csv2cwms/examples/complete_config.json | 19 +++ .../csv2cwms/tests/data/sample_config.json | 8 +- .../commands/csv2cwms/tests/test_dateutils.py | 3 +- .../csv2cwms/tests/test_expressions.py | 3 +- .../commands/csv2cwms/tests/test_fileio.py | 7 +- cwmscli/commands/csv2cwms/utils/dateutils.py | 26 ++++- cwmscli/commands/csv2cwms/utils/logging.py | 5 + 10 files changed, 158 insertions(+), 104 deletions(-) create mode 100644 cwmscli/commands/csv2cwms/examples/complete_config.json diff --git a/cwmscli/commands/commands_cwms.py b/cwmscli/commands/commands_cwms.py index cec79da..91510d6 100644 --- a/cwmscli/commands/commands_cwms.py +++ b/cwmscli/commands/commands_cwms.py @@ -36,11 +36,11 @@ def shefcritimport(filename, office, api_root, api_key, api_key_loc): @click.command("csv2cwms", help="Store CSV TimeSeries data to CWMS using a config file") @common_api_options @click.option( - "-l", - "--location", + "--input-keys", + "input_keys", default="all", show_default=True, - help='Location ID. Use "-p=all" for all locations.', + help='Input keys. Defaults to all keys/files with --input-keys=all. These are the keys under "input_files" in a given config file. This option lets you run a single file from a config that contains multiple files. Example: --input-keys=file1', ) @click.option( "-lb", @@ -67,15 +67,6 @@ def shefcritimport(filename, office, api_root, api_key, api_key_loc): help="Override CSV file (else use config)", ) @click.option("--log", show_default=True, help="Path to the log file.") -@click.option( - "-dp", - "--data-path", - "data_path", - default=".", - show_default=True, - type=click.Path(exists=True, file_okay=False), - help="Directory where csv files are stored", -) @click.option("--dry-run", is_flag=True, help="Log only (no HTTP calls)") @click.option("--begin", type=str, help="YYYY-MM-DDTHH:MM (local to --tz)") @click.option("-tz", "--timezone", "tz", default="GMT", show_default=True) diff --git a/cwmscli/commands/csv2cwms/README.md b/cwmscli/commands/csv2cwms/README.md index 96a4594..7e62d5b 100644 --- a/cwmscli/commands/csv2cwms/README.md +++ b/cwmscli/commands/csv2cwms/README.md @@ -8,36 +8,44 @@ To View the Help: `cwms-cli csv2cwms --help` Usage: cwms-cli csv2cwms [OPTIONS] - Store CSV TimeSeries data to CWMS using a config file +Store CSV TimeSeries data to CWMS using a config file Options: - -o, --office TEXT Office to grab data for [required] - -a, --api_root TEXT Api Root for CDA. Can be user defined or placed - in a env variable CDA_API_ROOT [required] - -k, --api_key TEXT api key for CDA. Can be user defined or place in - env variable CDA_API_KEY. one of api_key or - api_key_loc are required - -l, --location TEXT Location ID. Use "-p=all" for all locations. - [default: all] - -lb, --lookback INTEGER Lookback period in HOURS [default: 120] - -v, --verbose Verbose logging - -c, --config PATH Path to JSON config file [required] - [default: all] - -lb, --lookback INTEGER Lookback period in HOURS [default: 120] - -v, --verbose Verbose logging - [default: all] - [default: all] - -lb, --lookback INTEGER Lookback period in HOURS [default: 120] - -v, --verbose Verbose logging - -c, --config PATH Path to JSON config file [required] - -df, --data-file TEXT Override CSV file (else use config) - --log TEXT Path to the log file. - -dp, --data-path DIRECTORY Directory where csv files are stored [default: - .] - --dry-run Log only (no HTTP calls) - --begin TEXT YYYY-MM-DDTHH:MM (local to --tz) - -tz, --timezone TEXT [default: GMT] - --ignore-ssl-errors Ignore TLS errors (testing only) - --version Show the version and exit. - --help Show this message and exit. +-o, --office TEXT Office to grab data for [required] +-a, --api_root TEXT Api Root for CDA. Can be user defined or placed +in a env variable CDA_API_ROOT [required] +-k, --api_key TEXT api key for CDA. Can be user defined or place in +env variable CDA_API_KEY. one of api_key or +api_key_loc are required +-l, --location TEXT Location ID. Use "-p=all" for all locations. +[default: all] +-lb, --lookback INTEGER Lookback period in HOURS [default: 120] +-v, --verbose Verbose logging +-c, --config PATH Path to JSON config file [required] +[default: all] +-lb, --lookback INTEGER Lookback period in HOURS [default: 120] +-v, --verbose Verbose logging +[default: all] +[default: all] +-lb, --lookback INTEGER Lookback period in HOURS [default: 120] +-v, --verbose Verbose logging +-c, --config PATH Path to JSON config file [required] +-df, --data-file TEXT Override CSV file (else use config) +--log TEXT Path to the log file. +-dp, --data-path DIRECTORY Directory where csv files are stored [default: +.] +--dry-run Log only (no HTTP calls) +--begin TEXT YYYY-MM-DDTHH:MM (local to --tz) +-tz, --timezone TEXT [default: GMT] +--ignore-ssl-errors Ignore TLS errors (testing only) +--version Show the version and exit. +--help Show this message and exit. +## Features + +- Allow for specifying one or more date formats that might be seen per input csv file +- Allow mathematical operations across multiple columns and storing into one timeseries +- Store one column of data with a user-specified precision and units to a timeseries identifier +- Dry runs to test what data might look like prior to database storage +- Verbose logging via the -v flag +- Colored terminal output for user readability diff --git a/cwmscli/commands/csv2cwms/__main__.py b/cwmscli/commands/csv2cwms/__main__.py index 358a96d..1ec54c5 100644 --- a/cwmscli/commands/csv2cwms/__main__.py +++ b/cwmscli/commands/csv2cwms/__main__.py @@ -1,5 +1,4 @@ # Script Entry File -import json import os import sys import time @@ -51,7 +50,6 @@ API_KEY = os.getenv("CDA_API_KEY") OFFICE = os.getenv("CDA_OFFICE", "SWT") HOST = os.getenv("CDA_HOST") -LOOKBACK_DAYS = int(os.getenv("CDA_LOOKBACK_DAYS", 5)) # Default to 5 days if not set if [API_KEY, OFFICE, HOST].count(None) > 0: raise ValueError( @@ -59,39 +57,33 @@ ) -def parse_file(file_path, begin_time, lookback, timezone="GMT"): +def parse_file(file_path, begin_time, date_format, timezone="GMT"): csv_data = load_csv(file_path) header = csv_data[0] data = csv_data[1:] ts_data = {} - lookback_datetime = begin_time - timedelta(hours=lookback) logger.debug(f"Begin time: {begin_time}") - logger.debug(f"Lookback datetime: {lookback_datetime}") for row in data: # Skip empty rows or rows without a timestamp if not row: continue - row_datetime = parse_date(row[0], tz_str=timezone) - # Skip rows that are before/older than the lookback period and after the begin time - logger.debug(f"Row datetime: {row_datetime}") - if row_datetime < lookback_datetime or row_datetime > begin_time: - continue + row_datetime = parse_date(row[0], tz_str=timezone, date_format=date_format) # Guarantee only one entry per timestamp ts_data[int(row_datetime.timestamp())] = row return {"header": header, "data": ts_data} -def load_timeseries(file_data, project, config): +def load_timeseries(file_data, file_key, config): header = file_data.get("header", []) data = file_data.get("data", {}) if not header or not data: raise ValueError( - "No data found in the CSV file for the range selected: check the --lookback period and/or --begin time. You will also want to ensure you set the timezone of the CSV file with --tz America/Chicago or similar." + "No data found in the CSV file for the range selected. Please ensure you set the timezone of the CSV file with --tz America/Chicago or similar." ) - ts_config = config["projects"][project]["timeseries"] - project_ts = [] + ts_config = config["input_files"][file_key]["timeseries"] + file_ts = [] # Interval in seconds interval = config.get("interval") @@ -138,9 +130,9 @@ def load_timeseries(file_data, project, config): logger.debug( f"Timeseries {name} data range: {colorize(datetime.fromtimestamp(start_epoch), 'blue')} to {colorize(datetime.fromtimestamp(end_epoch), 'blue')}" ) - project_ts.append(ts_obj) + file_ts.append(ts_obj) - return project_ts + return file_ts def config_check(config): @@ -149,20 +141,25 @@ def config_check(config): logger.warning( "Configuration file does not contain an 'interval' key (and value in seconds), this is recommended per CSV file to avoid ambiguity." ) - if not config.get("projects"): - raise ValueError("Configuration file must contain a 'projects' key.") - for proj, proj_data in config.get("projects").items(): - # Only check the specified project or if all projects are specified - if proj != "all" and proj != proj.lower(): + if config.get("projects"): + logger.warning( + "Configuration file contains a 'projects' key, this has been renamed to 'input_files' for clarity. Continuing for backwards compatibility." + ) + config["input_files"] = config.pop("projects") + if not config.get("input_files"): + raise ValueError("Configuration file must contain an 'input_files' key.") + for file_key, file_data in config.get("input_files").items(): + # Only check the specified keys or if all keys are specified + if file_key != "all" and file_key != file_key.lower(): continue - if not proj_data.get("timeseries"): + if not file_data.get("timeseries"): raise ValueError( - f"Configuration file must contain a 'timeseries' key for project '{proj}'." + f"Configuration file must contain a 'timeseries' key for file '{file_key}'." ) - for ts_name, ts_data in proj_data.get("timeseries").items(): + for ts_name, ts_data in file_data.get("timeseries").items(): if not ts_data.get("columns"): raise ValueError( - f"Configuration file must contain a 'columns' key for timeseries '{ts_name}' in project '{proj}'." + f"Configuration file must contain a 'columns' key for timeseries '{ts_name}' in file '{file_key}'." ) @@ -190,7 +187,6 @@ def main(*args, **kwargs): setup_logger(kwargs.get("log"), verbose=kwargs.get("verbose")) logger.info(f"Begin time: {begin_time}") logger.debug(f"Timezone: {tz}") - logger.debug(f"Lookback period: {kwargs.get("lookback")} hours") # Override environment variables if provided in CLI if kwargs.get("coop"): HOST = os.getenv("CDA_COOP_HOST") @@ -198,63 +194,67 @@ def main(*args, **kwargs): raise ValueError( "Environment variable CDA_COOP_HOST must be set to use --coop flag." ) - config = read_config(kwargs.get("config_path")) + config_path = kwargs.get("config_path") + config = read_config(config_path) config_check(config) - PROJECTS = config.get("projects") - # Override projects if one is specified in CLI - if kwargs.get("project"): - if kwargs.get("project") == "all": - PROJECTS = config.get("projects", {}).keys() + INPUT_FILES = config.get("input_files", {}) + # Override file names if one is specified in CLI + if kwargs.get("input_keys"): + if kwargs.get("input_keys") == "all": + INPUT_FILES = config.get("input_files", {}).keys() else: - PROJECTS = [kwargs.get("project")] - if not PROJECTS: - raise ValueError("Configuration file must contain a 'projects' key.") - logger.info(f"Started for {','.join(PROJECTS)} projects.") + INPUT_FILES = kwargs.get("input_keys").split(",") + logger.info(f"Started for {','.join(INPUT_FILES)} input files.") # Input checks - # if kwargs.get("project") != "all" and kwargs.get("project") not in PROJECTS: + # if kwargs.get("file_name") != "all" and kwargs.get("file_name") not in INPUT_FILES: # raise ValueError( - # f"Invalid project name '{kwargs.get("project")}'. Valid options are: {', '.join(PROJECTS)}" + # f"Invalid file name '{kwargs.get("file_name")}'. Valid options are: {', '.join(INPUT_FILES)}" # ) - if kwargs.get("lookback") < 0: - raise ValueError("Lookback period must be a non-negative integer.") - - # Loop the projects and post the data - for proj in PROJECTS: - HYDRO_DIR = config.get("projects", {}).get(proj, {}).get("dir", "") - # Check if the user wants to override the data file name from what is in the config - DATA_FILE = kwargs.get("data_file") or config.get("projects", {}).get( - proj, {} - ).get("file", "") + # Loop the file names and post the data + for file_name in INPUT_FILES: + # Grab the csv file path from the config + CONFIG_ITEM = config.get("input_files", {}).get(file_name, {}) + DATA_FILE = CONFIG_ITEM.get("data_path", "") if not DATA_FILE: logger.warning( - f"No data file specified for project '{proj}'. {colorize(f'Skipping {proj}', 'red')}. Please provide a valid CSV file path using --data_file or ensure the 'file' key is set in the config." + # TODO: List URL to example in doc site once available + f"No data file specified for input-keys '{file_name}' in {config_path}. {colorize(f'Skipping {file_name}', 'red')}. Please provide a valid CSV file path by ensuring the 'data_path' key is set in the config." ) continue csv_data = parse_file( - os.path.join(kwargs.get("data_path"), HYDRO_DIR, DATA_FILE), + DATA_FILE, begin_time, - kwargs.get("lookback"), + CONFIG_ITEM.get("date_format"), kwargs.get("tz"), ) - ts_min_data = load_timeseries(csv_data, proj, config) + try: + ts_min_data = load_timeseries(csv_data, file_name, config) + except ValueError as e: + logger.error(f"Error loading timeseries for {file_name}: {e}") + continue if kwargs.get("dry_run"): logger.info("DRY RUN enabled. No data will be posted") for ts_object in ts_min_data: try: ts_object.update({"office-id": kwargs.get("office")}) + logger.info( + "Store Rule: " + CONFIG_ITEM.get("store_rule", "") + if CONFIG_ITEM.get("store_rule", "") + else f"No Store Rule specified, will default to REPLACE_ALL in {config_path}." + ) if kwargs.get("dry_run"): logger.info(f"DRY RUN: {ts_object}") else: cwms.store_timeseries( data=ts_object, - store_rule=kwargs.get("store_rule", "REPLACE_ALL"), + store_rule=CONFIG_ITEM.get("store_rule", "REPLACE_ALL"), ) logger.info(f"Stored {ts_object['name']} values") except Exception as e: logger.error( - f"Error posting data for {proj}: {e}\n{traceback.format_exc()}" + f"Error posting data for {file_name}: {e}\n{traceback.format_exc()}" ) logger.debug(f"\tExecution time: {round(time.time() - start_time, 3)} seconds.") diff --git a/cwmscli/commands/csv2cwms/examples/complete_config.json b/cwmscli/commands/csv2cwms/examples/complete_config.json new file mode 100644 index 0000000..b117cb9 --- /dev/null +++ b/cwmscli/commands/csv2cwms/examples/complete_config.json @@ -0,0 +1,19 @@ +{ + "interval": 3600, + "input_files": { + "BROK": { + "data_path": "cwmscli/commands/csv2cwms/tests/data/sample_brok.csv", + "date_format": [ + "%m/%d/%Y %H:%M:%S", + "%m/%d/%Y %H:%M" + ], + "timeseries": { + "BROK.Elev.Inst.15Minutes.0.Rev-SCADA-cda": { + "columns": "Headwater", + "units": "ft", + "precision": 2 + } + } + } + } +} \ No newline at end of file diff --git a/cwmscli/commands/csv2cwms/tests/data/sample_config.json b/cwmscli/commands/csv2cwms/tests/data/sample_config.json index bebd434..70a96a0 100644 --- a/cwmscli/commands/csv2cwms/tests/data/sample_config.json +++ b/cwmscli/commands/csv2cwms/tests/data/sample_config.json @@ -1,7 +1,13 @@ { "interval": null, - "projects": { + "input_files": { "BROK": { + "data_path": "cwmscli/commands/csv2cwms/tests/data/sample_brok.csv", + "store_rule": "REPLACE_ALL", + "date_format": [ + "%m/%d/%Y %H:%M:%S", + "%m/%d/%Y %H:%M" + ], "timeseries": { "BROK.Elev.Inst.15Minutes.0.Rev-SCADA-cda": { "columns": "Headwater", diff --git a/cwmscli/commands/csv2cwms/tests/test_dateutils.py b/cwmscli/commands/csv2cwms/tests/test_dateutils.py index 21785c8..cd75287 100644 --- a/cwmscli/commands/csv2cwms/tests/test_dateutils.py +++ b/cwmscli/commands/csv2cwms/tests/test_dateutils.py @@ -1,7 +1,8 @@ from datetime import datetime, timedelta import pytest -from utils.dateutils import determine_interval, parse_date, safe_zoneinfo + +from ..utils.dateutils import determine_interval, parse_date, safe_zoneinfo def test_parse_date_valid_formats(): diff --git a/cwmscli/commands/csv2cwms/tests/test_expressions.py b/cwmscli/commands/csv2cwms/tests/test_expressions.py index 964793f..eb70d5b 100644 --- a/cwmscli/commands/csv2cwms/tests/test_expressions.py +++ b/cwmscli/commands/csv2cwms/tests/test_expressions.py @@ -1,5 +1,6 @@ import pytest -from utils.expression import eval_expression + +from ..utils.expression import eval_expression @pytest.mark.parametrize( diff --git a/cwmscli/commands/csv2cwms/tests/test_fileio.py b/cwmscli/commands/csv2cwms/tests/test_fileio.py index 5a2ebb3..073d5cf 100644 --- a/cwmscli/commands/csv2cwms/tests/test_fileio.py +++ b/cwmscli/commands/csv2cwms/tests/test_fileio.py @@ -1,7 +1,8 @@ import os import pytest -from utils.fileio import load_csv, read_config + +from ..utils.fileio import load_csv, read_config def test_load_csv_valid(): @@ -31,8 +32,8 @@ def test_read_config_valid(): path = os.path.join(os.path.dirname(__file__), "data", "sample_config.json") config = read_config(path) assert isinstance(config, dict) - assert "projects" in config - assert "BROK" in config["projects"] + assert "input_files" in config + assert "BROK" in config["input_files"] def test_read_config_invalid_json(tmp_path): diff --git a/cwmscli/commands/csv2cwms/utils/dateutils.py b/cwmscli/commands/csv2cwms/utils/dateutils.py index 9deb7bc..c2410d3 100644 --- a/cwmscli/commands/csv2cwms/utils/dateutils.py +++ b/cwmscli/commands/csv2cwms/utils/dateutils.py @@ -1,3 +1,4 @@ +import logging from collections import Counter from datetime import datetime, timezone from typing import List @@ -9,6 +10,8 @@ ZoneInfo = None ZoneInfoNotFoundError = Exception +logger = logging.getLogger(__name__) + DATE_STRINGS = [ "%m/%d/%Y %H:%M:%S", "%m/%d/%Y %H:%M", @@ -35,7 +38,7 @@ def safe_zoneinfo(key: str): return timezone.utc -def parse_date(date, tz_str="UTC") -> datetime: +def parse_date(date, tz_str="UTC", date_format: str = "") -> datetime: """Handle all date types seen in hydropower files NOTE: TimeZone naive - assumes all timestamps are in the same timezone Args: @@ -44,9 +47,28 @@ def parse_date(date, tz_str="UTC") -> datetime: if isinstance(date, int): return datetime.fromtimestamp(date, tz=safe_zoneinfo(tz_str)) - for fmt in DATE_STRINGS: + if isinstance(date_format, str): + # Handle comma-separated list of formats + if date_format.find(",") >= 0: + date_format = [fmt.strip() for fmt in date_format.split(",") if fmt.strip()] + date_format = [date_format] + + # Include the user-specified date format first, if provided + for idx, fmt in enumerate(date_format + DATE_STRINGS): try: + if not fmt: + continue dt_naive = datetime.strptime(date, fmt) + if idx > 0: + # Only log if using a fallback format + if not date_format: + logger.warning( + f"Using fallback date format '{fmt}' for date '{date}'. No user-specified format was provided." + ) + else: + logger.warning( + f"Using fallback date format '{fmt}' for date '{date}'. The user-specified format is '{date_format}'." + ) return dt_naive.replace(tzinfo=safe_zoneinfo(tz_str)) except ValueError: continue diff --git a/cwmscli/commands/csv2cwms/utils/logging.py b/cwmscli/commands/csv2cwms/utils/logging.py index 340ea67..0726b88 100644 --- a/cwmscli/commands/csv2cwms/utils/logging.py +++ b/cwmscli/commands/csv2cwms/utils/logging.py @@ -50,6 +50,11 @@ def setup_logger( Returns: logger: logging.Logger """ + + # Remove the default logger handlers from cwms-cli so we can set up our own + root = logging.getLogger() + for h in root.handlers[:]: + root.removeHandler(h) # Create formatter and attach to handler formatter = ColorFormatter( "[%(asctime)s] [%(levelname)s] %(message)s", "%Y-%m-%d %H:%M:%S" From de5140f7714186da1d43c65be1169b7bcba060af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Sep 2025 22:13:23 -0500 Subject: [PATCH 18/25] Bump actions/checkout from 4 to 5 (#36) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5.
Release notes

Sourced from actions/checkout's releases.

v5.0.0

What's Changed

⚠️ Minimum Compatible Runner Version

v2.327.1
Release Notes

Make sure your runner is updated to this version or newer to use this release.

Full Changelog: https://github.com/actions/checkout/compare/v4...v5.0.0

v4.3.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v4...v4.3.0

v4.2.2

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v4.2.1...v4.2.2

v4.2.1

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v4.2.0...v4.2.1

... (truncated)

Changelog

Sourced from actions/checkout's changelog.

Changelog

V5.0.0

V4.3.0

v4.2.2

v4.2.1

v4.2.0

v4.1.7

v4.1.6

v4.1.5

v4.1.4

v4.1.3

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/checkout&package-manager=github_actions&previous-version=4&new-version=5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ba5fa82..0819992 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,7 +10,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: actions/setup-python@v5 with: python-version: "3.12" From bc61b0d4067de0aebb4c3ce393992b04ee20a87d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Sep 2025 22:35:40 -0500 Subject: [PATCH 19/25] Bump actions/setup-python from 5 to 6 (#37) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [//]: # (dependabot-start) ⚠️ **Dependabot is rebasing this PR** ⚠️ Rebasing might not happen immediately, so don't worry if this takes some time. Note: if you make any changes to this PR yourself, they will take precedence over the rebase. --- [//]: # (dependabot-end) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6.
Release notes

Sourced from actions/setup-python's releases.

v6.0.0

What's Changed

Breaking Changes

Make sure your runner is on version v2.327.1 or later to ensure compatibility with this release. See Release Notes

Enhancements:

Bug fixes:

Dependency updates:

New Contributors

Full Changelog: https://github.com/actions/setup-python/compare/v5...v6.0.0

v5.6.0

What's Changed

Full Changelog: https://github.com/actions/setup-python/compare/v5...v5.6.0

v5.5.0

What's Changed

Enhancements:

Bug fixes:

... (truncated)

Commits
  • e797f83 Upgrade to node 24 (#1164)
  • 3d1e2d2 Revert "Enhance cache-dependency-path handling to support files outside the w...
  • 65b0712 Clarify pythonLocation behavior for PyPy and GraalPy in environment variables...
  • 5b668cf Bump actions/checkout from 4 to 5 (#1181)
  • f62a0e2 Change missing cache directory error to warning (#1182)
  • 9322b3c Upgrade setuptools to 78.1.1 to fix path traversal vulnerability in PackageIn...
  • fbeb884 Bump form-data to fix critical vulnerabilities #182 & #183 (#1163)
  • 03bb615 Bump idna from 2.9 to 3.7 in /tests/data (#843)
  • 36da51d Add version parsing from Pipfile (#1067)
  • 3c6f142 update documentation (#1156)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-python&package-manager=github_actions&previous-version=5&new-version=6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Charles Graham --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0819992..3f42a22 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.12" - name: Install deps From c5cc51d937f140086b0e5b139eb482d799d6d60d Mon Sep 17 00:00:00 2001 From: Charles Graham Date: Tue, 23 Sep 2025 16:55:30 -0500 Subject: [PATCH 20/25] Combine USGS Scripts into Group (#40) Combines the various USGS scripts into a click group allowing for subcommands Base Command ```bash > cwms-cli Usage: cwms-cli [OPTIONS] COMMAND [ARGS]... Options: --help Show this message and exit. Commands: blob Manage CWMS Blobs (upload, download, delete, update, list) csv2cwms Store CSV TimeSeries data to CWMS using a config file shefcritimport Import SHEF crit file into timeseries group for SHEF... usgs USGS utilities ``` USGS Command ```bash > cwms-cli usgs Usage: cwms-cli usgs [OPTIONS] COMMAND [ARGS]... USGS utilities Options: --help Show this message and exit. Commands: measurements Store USGS measurements into CWMS database ratings Get USGS ratings and store into CWMS database ratings-ini-file-import Store rating ini file information into... timeseries Get USGS timeseries values and store into CWMS... ``` Measurements Command ```bash > cwms-cli usgs measurements --help Usage: cwms-cli usgs measurements [OPTIONS] Store USGS measurements into CWMS database Options: -d, --days_back_modified TEXT Days back from current time measurements have been modified in USGS database. Can be integer value -c, --days_back_collected TEXT Days back from current time measurements have been collected. Can be integer value -o, --office TEXT Office to grab data for [required] -a, --api_root TEXT Api Root for CDA. Can be user defined or placed in a env variable CDA_API_ROOT [required] -k, --api_key TEXT api key for CDA. Can be user defined or place in env variable CDA_API_KEY. one of api_key or api_key_loc are required -kl, --api_key_loc TEXT file storing Api Key. One of api_key or api_key_loc are required -b, --backfill TEXT Backfill POR data, use list of USGS IDs (e.g. 05057200, 05051300) or the word 'group' to attempt to backfill all sites in the OFFICE id's Data Acquisition->USGS Measurements group --help Show this message and exit. ``` --- cwmscli/__main__.py | 7 ++-- .../commands_getusgs.py => usgs/__init__.py} | 32 ++++++++++++------- .../getUSGS_ratings_cda.py} | 5 +++ .../getugsg_cda.py => usgs/getusgs_cda.py} | 0 .../getusgs_measurements_cda.py | 1 - .../rating_ini_file_import.py | 0 6 files changed, 28 insertions(+), 17 deletions(-) rename cwmscli/{getusgs/commands_getusgs.py => usgs/__init__.py} (81%) rename cwmscli/{getusgs/getUSGS_ratings_CDA.py => usgs/getUSGS_ratings_cda.py} (98%) rename cwmscli/{getusgs/getugsg_cda.py => usgs/getusgs_cda.py} (100%) rename cwmscli/{getusgs => usgs}/getusgs_measurements_cda.py (99%) rename cwmscli/{getusgs => usgs}/rating_ini_file_import.py (100%) diff --git a/cwmscli/__main__.py b/cwmscli/__main__.py index c13df87..730e1c9 100644 --- a/cwmscli/__main__.py +++ b/cwmscli/__main__.py @@ -1,8 +1,8 @@ import click from cwmscli.commands import commands_cwms -from cwmscli.getusgs import commands_getusgs from cwmscli.reporting import reporting_cli +from cwmscli.usgs import usgs_group @click.group() @@ -10,10 +10,7 @@ def cli(): pass -cli.add_command(commands_getusgs.getusgs_timeseries) -cli.add_command(commands_getusgs.getusgs_ratings) -cli.add_command(commands_getusgs.ratingsinifileimport) -cli.add_command(commands_getusgs.getusgs_measurements) +cli.add_command(usgs_group, name="usgs") cli.add_command(commands_cwms.shefcritimport) cli.add_command(commands_cwms.csv2cwms_cmd) cli.add_command(commands_cwms.blob_group) diff --git a/cwmscli/getusgs/commands_getusgs.py b/cwmscli/usgs/__init__.py similarity index 81% rename from cwmscli/getusgs/commands_getusgs.py rename to cwmscli/usgs/__init__.py index adbf7dc..249abb1 100644 --- a/cwmscli/getusgs/commands_getusgs.py +++ b/cwmscli/usgs/__init__.py @@ -1,5 +1,17 @@ import click +from cwmscli import requirements as reqs +from cwmscli.utils.deps import requires + + +@click.group() +def usgs_group(): + """USGS utilities""" + pass + + +import click + from cwmscli import requirements as reqs from cwmscli.utils import ( api_key_loc_option, @@ -19,8 +31,8 @@ ) -@click.command( - "getusgs-timeseries", help="Get USGS timeseries values and store into CWMS database" +@usgs_group.command( + "timeseries", help="Get USGS timeseries values and store into CWMS database" ) @office_option @days_back_option @@ -29,7 +41,7 @@ @api_key_loc_option @requires(reqs.cwms, reqs.requests) def getusgs_timeseries(office, days_back, api_root, api_key, api_key_loc): - from cwmscli.getusgs.getugsg_cda import getusgs_cda + from cwmscli.usgs.getusgs_cda import getusgs_cda api_key = get_api_key(api_key, api_key_loc) getusgs_cda( @@ -40,7 +52,7 @@ def getusgs_timeseries(office, days_back, api_root, api_key, api_key_loc): ) -@click.command("getusgs-ratings", help="Get USGS ratings and store into CWMS database") +@usgs_group.command("ratings", help="Get USGS ratings and store into CWMS database") @office_option @days_back_option @api_root_option @@ -48,7 +60,7 @@ def getusgs_timeseries(office, days_back, api_root, api_key, api_key_loc): @api_key_loc_option @requires(reqs.cwms, reqs.requests, reqs.dataretrieval) def getusgs_ratings(office, days_back, api_root, api_key, api_key_loc): - from cwmscli.getusgs.getUSGS_ratings_CDA import getusgs_rating_cda + from cwmscli.usgs.getUSGS_ratings_cda import getusgs_rating_cda api_key = get_api_key(api_key, api_key_loc) getusgs_rating_cda( @@ -59,7 +71,7 @@ def getusgs_ratings(office, days_back, api_root, api_key, api_key_loc): ) -@click.command( +@usgs_group.command( "ratings-ini-file-import", help="Store rating ini file information into database to be used with getusgs_ratings", ) @@ -75,15 +87,13 @@ def getusgs_ratings(office, days_back, api_root, api_key, api_key_loc): @api_key_loc_option @requires(reqs.cwms, reqs.requests) def ratingsinifileimport(filename, api_root, api_key, api_key_loc): - from cwmscli.getusgs.rating_ini_file_import import rating_ini_file_import + from cwmscli.usgs.rating_ini_file_import import rating_ini_file_import api_key = get_api_key(api_key, api_key_loc) rating_ini_file_import(api_root=api_root, api_key=api_key, ini_filename=filename) -@click.command( - "getusgs-measurements", help="Store USGS measurements into CWMS database" -) +@usgs_group.command("measurements", help="Store USGS measurements into CWMS database") @click.option( "-d", "--days_back_modified", @@ -117,7 +127,7 @@ def getusgs_measurements( api_key_loc, backfill, ): - from cwmscli.getusgs.getusgs_measurements_cda import getusgs_measurement_cda + from cwmscli.usgs.getusgs_measurements_cda import getusgs_measurement_cda backfill_group = False backfill_list = False diff --git a/cwmscli/getusgs/getUSGS_ratings_CDA.py b/cwmscli/usgs/getUSGS_ratings_cda.py similarity index 98% rename from cwmscli/getusgs/getUSGS_ratings_CDA.py rename to cwmscli/usgs/getUSGS_ratings_cda.py index 3fd9368..238b167 100644 --- a/cwmscli/getusgs/getUSGS_ratings_CDA.py +++ b/cwmscli/usgs/getUSGS_ratings_cda.py @@ -1,4 +1,5 @@ import logging +import sys from datetime import datetime, timedelta from json import loads @@ -61,6 +62,10 @@ def get_rating_ids_from_specs(office_id): rating_specs = cwms.get_rating_specs(office_id=office_id).df if "effective-dates" not in rating_specs.columns: rating_specs["effective-dates"] = np.nan + # Determine if any specs return + if rating_specs.empty: + logging.warning(f"No rating specifications found for office {office_id}") + sys.exit() rating_specs = rating_specs.dropna(subset=["description"]) for rating_type in rating_types: rating_specs.loc[ diff --git a/cwmscli/getusgs/getugsg_cda.py b/cwmscli/usgs/getusgs_cda.py similarity index 100% rename from cwmscli/getusgs/getugsg_cda.py rename to cwmscli/usgs/getusgs_cda.py diff --git a/cwmscli/getusgs/getusgs_measurements_cda.py b/cwmscli/usgs/getusgs_measurements_cda.py similarity index 99% rename from cwmscli/getusgs/getusgs_measurements_cda.py rename to cwmscli/usgs/getusgs_measurements_cda.py index a4a4eb6..598d082 100644 --- a/cwmscli/getusgs/getusgs_measurements_cda.py +++ b/cwmscli/usgs/getusgs_measurements_cda.py @@ -9,7 +9,6 @@ import pytz import requests from dataretrieval import nwis -from dotenv import load_dotenv # --- Constants --- CWMS_MISSING_VALUE = -340282346638528859811704183484516925440 diff --git a/cwmscli/getusgs/rating_ini_file_import.py b/cwmscli/usgs/rating_ini_file_import.py similarity index 100% rename from cwmscli/getusgs/rating_ini_file_import.py rename to cwmscli/usgs/rating_ini_file_import.py From f337d3f88375c7dea84206c5fe0896504e376886 Mon Sep 17 00:00:00 2001 From: Charles Graham Date: Tue, 23 Sep 2025 17:29:05 -0500 Subject: [PATCH 21/25] Update .readthedocs.yaml (#43) Fix RTD error --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 51d46cb..b3f0062 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -13,4 +13,4 @@ python: - method: pip path: . - requirements: docs/requirements.txt -formats: [html, pdf] +formats: [pdf] From d5441e43bb0fb7f62ddb0f0283a7eb056ba3b057 Mon Sep 17 00:00:00 2001 From: Eric Novotny Date: Wed, 24 Sep 2025 11:57:42 -0500 Subject: [PATCH 22/25] revert back to 3.9 (#42) read the docs packages should load from the requirement.txt file in the docs folder. and python version set from .readthedocs.yaml. --- pyproject.toml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e1f8401..3a78e26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ keywords = ["USACE", "CWMS"] authors = ["Hydrologic Engineering Center"] [tool.poetry.dependencies] -python = "^3.11" +python = "^3.9" click = "^8.1.8" hecdss = { version = ">=0.1.24", optional = true } # Via https://github.com/HydrologicEngineeringCenter/hec-python-library/blob/main/hec/shared.py#L9-10 cwms-python = { version = ">=0.8.0", optional = true} @@ -33,12 +33,6 @@ pre-commit = "^3.6.2" #pandas-stubs = "^2.2.1.240316" yamlfix = "^1.16.0" - -[tool.poetry.group.docs.dependencies] -sphinx = "^8.2.3" -sphinx-rtd-theme = "^3.0.2" -sphinx-click = "^6.1.0" - [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" From 1ba1a9c6ebb760d47d431ec5b9591064d94f4680 Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Mon, 6 Oct 2025 21:18:46 +0000 Subject: [PATCH 23/25] Fix global imports for lazy loading, fixes #47 --- cwmscli/reporting/__init__.py | 3 ++- cwmscli/reporting/config.py | 3 ++- cwmscli/reporting/core.py | 13 ++++++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/cwmscli/reporting/__init__.py b/cwmscli/reporting/__init__.py index b00a8f5..2b73cfb 100644 --- a/cwmscli/reporting/__init__.py +++ b/cwmscli/reporting/__init__.py @@ -6,7 +6,6 @@ from typing import Any, Dict, List, Optional import click -import cwms from cwmscli.reporting.config import Config from cwmscli.reporting.core import build_report_table @@ -94,6 +93,8 @@ def _render_template( }, ) def reporting_cli(config_path, template_dir, template_name, out_path): + import cwms + cfg = Config.from_yaml(config_path) tz = cfg.time_zone or "UTC" diff --git a/cwmscli/reporting/config.py b/cwmscli/reporting/config.py index 6b6ca2b..abdab52 100644 --- a/cwmscli/reporting/config.py +++ b/cwmscli/reporting/config.py @@ -3,7 +3,6 @@ from typing import Any, Dict, List, Optional import click -import yaml from cwmscli.reporting.models import ( ColumnSpec, @@ -62,6 +61,8 @@ class Config: @staticmethod def from_yaml(path: str) -> "Config": + import yaml + with open(path, "r", encoding="utf-8") as f: raw = yaml.safe_load(f) or {} diff --git a/cwmscli/reporting/core.py b/cwmscli/reporting/core.py index 984f149..02e6ebd 100644 --- a/cwmscli/reporting/core.py +++ b/cwmscli/reporting/core.py @@ -4,8 +4,6 @@ from typing import Any, Dict, List, Optional import click -import cwms -import pandas as pd from cwmscli.reporting.config import Config from cwmscli.reporting.models import ProjectSpec @@ -27,7 +25,11 @@ def _fetch_multi_df( unit: str, begin: Optional[datetime], end: Optional[datetime], -) -> pd.DataFrame: +): + import pandas as pd + + import cwms + df = cwms.get_multi_timeseries_df( ts_ids=tsids, office_id=office, @@ -48,6 +50,8 @@ def _fetch_levels_dict( office: str, unit: str, ) -> Dict[str, float | None]: + import cwms + out: Dict[str, float | None] = {} for lvl in level_ids: try: @@ -94,6 +98,9 @@ def _format_value( def build_report_table( config: Config, begin: Optional[datetime], end: Optional[datetime] ) -> Dict[str, Any]: + import cwms + import pandas as pd + rows: List[str] = [p.location_id for p in config.projects] if not rows: raise click.UsageError("No 'projects' configured in YAML.") From 41b596fb192596f3fcc595e8b79edef259a9d899 Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Fri, 12 Dec 2025 21:04:42 +0000 Subject: [PATCH 24/25] Add requirement for yaml library for configs --- cwmscli/reporting/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cwmscli/reporting/__init__.py b/cwmscli/reporting/__init__.py index 2b73cfb..ddc8648 100644 --- a/cwmscli/reporting/__init__.py +++ b/cwmscli/reporting/__init__.py @@ -91,6 +91,12 @@ def _render_template( "version": "3.1.0", "desc": "Templating for pre/post-processing", }, + { + "module": "yaml", + "package": "PyYAML", + "version": "6.0", + "desc": "YAML parsing for report configuration", + }, ) def reporting_cli(config_path, template_dir, template_name, out_path): import cwms From 21dde5a41ab0393b2c6dc6e253f587f8810bf966 Mon Sep 17 00:00:00 2001 From: "Charles Graham, SWT" Date: Tue, 13 Jan 2026 21:47:11 +0000 Subject: [PATCH 25/25] Add all locations in --- cwmscli/reporting/configs/daily.yaml | 412 +++++++++++++-------------- 1 file changed, 206 insertions(+), 206 deletions(-) diff --git a/cwmscli/reporting/configs/daily.yaml b/cwmscli/reporting/configs/daily.yaml index 32a4de8..5c45c7c 100644 --- a/cwmscli/reporting/configs/daily.yaml +++ b/cwmscli/reporting/configs/daily.yaml @@ -1,206 +1,206 @@ -# daily.yaml -office: "SWT" -cda_api_root: "https://cwms-data.usace.army.mil/cwms-data/" - -# Either use a rolling window: -begin: "2025-09-16T00:00:00-05:00" -end: "2025-09-17T00:00:00-05:00" -# Or pin to a single instant, globally: -target_time: "0800 09/17/2025 America/Chicago" - -default_unit: "EN" -missing: "----" # used when no value at all was found -undefined: "--NA--" # used when a value exists but is NaN/invalid -time_zone: "America/Chicago" # default zone for parsing target_time if zone omitted - -report: - district: "Tulsa District SWT" - name: "Daily Morning Reservoir Report" - logo_left: "https://www.swt-wc.usace.army.mil/images/logos/usace-logo-color.svg" - logo_right: "https://www.swt-wc.usace.army.mil/images/logos/tulsa.png" - -projects: - # - location_id: "SKIA" - # href: "https://www.swt-wc.usace.army.mil/skia.lakepage.html" - # - location_id: "BROK" - # href: "https://www.swt-wc.usace.army.mil/brok.lakepage.html" - # - location_id: "ALTU" - # href: "https://www.swt-wc.usace.army.mil/ALTU.lakepage.html" - # - location_id: "ARBU" - # href: "https://www.swt-wc.usace.army.mil/ARBU.lakepage.html" - # - location_id: "ARCA" - # href: "https://www.swt-wc.usace.army.mil/ARCA.lakepage.html" - # - location_id: "BIGH" - # href: "https://www.swt-wc.usace.army.mil/BIGH.lakepage.html" - # - location_id: "BIRC" - # href: "https://www.swt-wc.usace.army.mil/BIRC.lakepage.html" - # - location_id: "BROK" - # href: "https://www.swt-wc.usace.army.mil/BROK.lakepage.html" - # - location_id: "CANT" - # href: "https://www.swt-wc.usace.army.mil/CANT.lakepage.html" - # - location_id: "CHEN" - # href: "https://www.swt-wc.usace.army.mil/CHEN.lakepage.html" - # - location_id: "FCOB" - # href: "https://www.swt-wc.usace.army.mil/FCOB.lakepage.html" - # - location_id: "COPA" - # href: "https://www.swt-wc.usace.army.mil/COPA.lakepage.html" - # - location_id: "COUN" - # href: "https://www.swt-wc.usace.army.mil/COUN.lakepage.html" - # - location_id: "DENI" - # href: "https://www.swt-wc.usace.army.mil/DENI.lakepage.html" - # - location_id: "ELDR" - # href: "https://www.swt-wc.usace.army.mil/ELDR.lakepage.html" - # - location_id: "ELKC" - # href: "https://www.swt-wc.usace.army.mil/ELKC.lakepage.html" - # - location_id: "EUFA" - # href: "https://www.swt-wc.usace.army.mil/EUFA.lakepage.html" - # - location_id: "FALL" - # href: "https://www.swt-wc.usace.army.mil/FALL.lakepage.html" - # - location_id: "FCOB" - # href: "https://www.swt-wc.usace.army.mil/FCOB.lakepage.html" - - location_id: "FGIB" - href: "https://www.swt-wc.usace.army.mil/FGIB.lakepage.html" - # - location_id: "FOSS" - # href: "https://www.swt-wc.usace.army.mil/FOSS.lakepage.html" - # - location_id: "FSUP" - # href: "https://www.swt-wc.usace.army.mil/FSUP.lakepage.html" - # - location_id: "GSAL" - # href: "https://www.swt-wc.usace.army.mil/GSAL.lakepage.html" - # - location_id: "HEYB" - # href: "https://www.swt-wc.usace.army.mil/HEYB.lakepage.html" - # - location_id: "HUDS" - # href: "https://www.swt-wc.usace.army.mil/HUDS.lakepage.html" - # - location_id: "HUGO" - # href: "https://www.swt-wc.usace.army.mil/HUGO.lakepage.html" - # - location_id: "HULA" - # href: "https://www.swt-wc.usace.army.mil/HULA.lakepage.html" - # - location_id: "JOHN" - # href: "https://www.swt-wc.usace.army.mil/JOHN.lakepage.html" - # - location_id: "KAWL" - # href: "https://www.swt-wc.usace.army.mil/KAWL.lakepage.html" - # - location_id: "KEMP" - # href: "https://www.swt-wc.usace.army.mil/KEMP.lakepage.html" - # - location_id: "KEYS" - # href: "https://www.swt-wc.usace.army.mil/KEYS.lakepage.html" - # - location_id: "MARI" - # href: "https://www.swt-wc.usace.army.mil/MARI.lakepage.html" - # - location_id: "MCGE" - # href: "https://www.swt-wc.usace.army.mil/MCGE.lakepage.html" - # - location_id: "MERE" - # href: "https://www.swt-wc.usace.army.mil/MERE.lakepage.html" - # - location_id: "OOLO" - # href: "https://www.swt-wc.usace.army.mil/OOLO.lakepage.html" - # - location_id: "PATM" - # href: "https://www.swt-wc.usace.army.mil/PATM.lakepage.html" - # - location_id: "PENS" - # href: "https://www.swt-wc.usace.army.mil/PENS.lakepage.html" - # - location_id: "PINE" - # href: "https://www.swt-wc.usace.army.mil/PINE.lakepage.html" - # - location_id: "SARD" - # href: "https://www.swt-wc.usace.army.mil/SARD.lakepage.html" - # - location_id: "SKIA" - # href: "https://www.swt-wc.usace.army.mil/SKIA.lakepage.html" - # - location_id: "TENK" - # href: "https://www.swt-wc.usace.army.mil/TENK.lakepage.html" - # - location_id: "THUN" - # href: "https://www.swt-wc.usace.army.mil/THUN.lakepage.html" - # - location_id: "TOMS" - # href: "https://www.swt-wc.usace.army.mil/TOMS.lakepage.html" - # - location_id: "TORO" - # href: "https://www.swt-wc.usace.army.mil/TORO.lakepage.html" - # - location_id: "WAUR" - # href: "https://www.swt-wc.usace.army.mil/WAUR.lakepage.html" - # - location_id: "WIST" - # href: "https://www.swt-wc.usace.army.mil/WIST.lakepage.html" - -header: - project: - text: "Reservoir" # text in the far-left header cell - classes: "" # optional extra classes - align: center # optional: left|center|right - rows: - # Header row 1 - - - - text: "Pool Elevation" - colspan: 2 - - text: "Pool Limits" - colspan: 2 - - text: "Pool Occupied Storage" - colspan: 3 - - text: "8AM Status (cfs)" - colspan: 3 - - text: "Previous 24Hr Average (cfs)" - colspan: 3 - - text: "Precip (in)" - align: center - # Header row 2 - - - text: "" - rowspan: 2 - - text: "8AM
Current" - rowspan: 2 - - text: "8AM
Prev" - rowspan: 2 - - text: "Top Of" - colspan: 2 - - text: "(%)" - rowspan: 2 - - text: "C/F" - rowspan: 2 - - text: "(ac-ft)" - rowspan: 2 - - text: "Total Release" - rowspan: 2 - - text: "Power Release" - rowspan: 2 - - text: "Inflow" - rowspan: 2 - - text: "Total Release" - rowspan: 2 - - text: "Power Release" - rowspan: 2 - - text: "Inflow" - rowspan: 2 - - text: "24Hr" - rowspan: 2 - # Header row 3 - - - text: "Pool Elev (ft)" - - text: "Tailwater (ft)" - -columns: - # Timeseries at a specific time - - title: "Pool Elev (ft)" - key: "elev_ts" - office: "SWT" - unit: "EN" - precision: 2 - tsid: "{project}.Elev.Inst.1Hour.0.Ccp-Rev" - # override the global target time for THIS column (optional): - begin: "today 0800" - end: "today 0800" - # per-column render strings (optional): - missing: "--" - undefined: "~~~~.~~" - href: "https://water.usace.army.mil/office/{office}/ts?name={tsid}" - - title: "Pool Elev (ft)" - key: "elev_ts" - office: "SWT" - unit: "EN" - precision: 2 - tsid: "{project}.Elev.Inst.1Hour.0.Ccp-Rev" - begin: "yesterday 0800 America/Chicago" - end: "yesterday 0800 America/Chicago" - # override the global target time for THIS column (optional): - target_time: "2025-09-21T08:00:00-05:00" - # per-column render strings (optional): - missing: "--" - undefined: "~~~~.~~" - href: "https://water.usace.army.mil/office/{office}/ts?name={tsid}" - # Level identifier (no sampling window; we just read the level) - - title: "Top of Conservation (ft)" - key: "top_cons" - office: "SWT" - unit: "ft" - precision: 2 - level: "{project}.Elev.Inst.0.Top of Conservation" - # optional per column overrides - undefined: "n/a" +# daily.yaml +office: "SWT" +cda_api_root: "https://cwms-data.usace.army.mil/cwms-data/" + +# Either use a rolling window: +begin: "2025-09-16T00:00:00-05:00" +end: "2025-09-17T00:00:00-05:00" +# Or pin to a single instant, globally: +target_time: "0800 09/17/2025 America/Chicago" + +default_unit: "EN" +missing: "----" # used when no value at all was found +undefined: "--NA--" # used when a value exists but is NaN/invalid +time_zone: "America/Chicago" # default zone for parsing target_time if zone omitted + +report: + district: "Tulsa District SWT" + name: "Daily Morning Reservoir Report" + logo_left: "https://www.swt-wc.usace.army.mil/images/logos/usace-logo-color.svg" + logo_right: "https://www.swt-wc.usace.army.mil/images/logos/tulsa.png" + +projects: + - location_id: "SKIA" + href: "https://www.swt-wc.usace.army.mil/skia.lakepage.html" + - location_id: "BROK" + href: "https://www.swt-wc.usace.army.mil/brok.lakepage.html" + - location_id: "ALTU" + href: "https://www.swt-wc.usace.army.mil/ALTU.lakepage.html" + - location_id: "ARBU" + href: "https://www.swt-wc.usace.army.mil/ARBU.lakepage.html" + - location_id: "ARCA" + href: "https://www.swt-wc.usace.army.mil/ARCA.lakepage.html" + - location_id: "BIGH" + href: "https://www.swt-wc.usace.army.mil/BIGH.lakepage.html" + - location_id: "BIRC" + href: "https://www.swt-wc.usace.army.mil/BIRC.lakepage.html" + - location_id: "BROK" + href: "https://www.swt-wc.usace.army.mil/BROK.lakepage.html" + - location_id: "CANT" + href: "https://www.swt-wc.usace.army.mil/CANT.lakepage.html" + - location_id: "CHEN" + href: "https://www.swt-wc.usace.army.mil/CHEN.lakepage.html" + - location_id: "FCOB" + href: "https://www.swt-wc.usace.army.mil/FCOB.lakepage.html" + - location_id: "COPA" + href: "https://www.swt-wc.usace.army.mil/COPA.lakepage.html" + - location_id: "COUN" + href: "https://www.swt-wc.usace.army.mil/COUN.lakepage.html" + - location_id: "DENI" + href: "https://www.swt-wc.usace.army.mil/DENI.lakepage.html" + - location_id: "ELDR" + href: "https://www.swt-wc.usace.army.mil/ELDR.lakepage.html" + - location_id: "ELKC" + href: "https://www.swt-wc.usace.army.mil/ELKC.lakepage.html" + - location_id: "EUFA" + href: "https://www.swt-wc.usace.army.mil/EUFA.lakepage.html" + - location_id: "FALL" + href: "https://www.swt-wc.usace.army.mil/FALL.lakepage.html" + - location_id: "FCOB" + href: "https://www.swt-wc.usace.army.mil/FCOB.lakepage.html" + - location_id: "FGIB" + href: "https://www.swt-wc.usace.army.mil/FGIB.lakepage.html" + - location_id: "FOSS" + href: "https://www.swt-wc.usace.army.mil/FOSS.lakepage.html" + - location_id: "FSUP" + href: "https://www.swt-wc.usace.army.mil/FSUP.lakepage.html" + - location_id: "GSAL" + href: "https://www.swt-wc.usace.army.mil/GSAL.lakepage.html" + - location_id: "HEYB" + href: "https://www.swt-wc.usace.army.mil/HEYB.lakepage.html" + - location_id: "HUDS" + href: "https://www.swt-wc.usace.army.mil/HUDS.lakepage.html" + - location_id: "HUGO" + href: "https://www.swt-wc.usace.army.mil/HUGO.lakepage.html" + - location_id: "HULA" + href: "https://www.swt-wc.usace.army.mil/HULA.lakepage.html" + - location_id: "JOHN" + href: "https://www.swt-wc.usace.army.mil/JOHN.lakepage.html" + - location_id: "KAWL" + href: "https://www.swt-wc.usace.army.mil/KAWL.lakepage.html" + - location_id: "KEMP" + href: "https://www.swt-wc.usace.army.mil/KEMP.lakepage.html" + - location_id: "KEYS" + href: "https://www.swt-wc.usace.army.mil/KEYS.lakepage.html" + - location_id: "MARI" + href: "https://www.swt-wc.usace.army.mil/MARI.lakepage.html" + - location_id: "MCGE" + href: "https://www.swt-wc.usace.army.mil/MCGE.lakepage.html" + - location_id: "MERE" + href: "https://www.swt-wc.usace.army.mil/MERE.lakepage.html" + - location_id: "OOLO" + href: "https://www.swt-wc.usace.army.mil/OOLO.lakepage.html" + - location_id: "PATM" + href: "https://www.swt-wc.usace.army.mil/PATM.lakepage.html" + - location_id: "PENS" + href: "https://www.swt-wc.usace.army.mil/PENS.lakepage.html" + - location_id: "PINE" + href: "https://www.swt-wc.usace.army.mil/PINE.lakepage.html" + - location_id: "SARD" + href: "https://www.swt-wc.usace.army.mil/SARD.lakepage.html" + - location_id: "SKIA" + href: "https://www.swt-wc.usace.army.mil/SKIA.lakepage.html" + - location_id: "TENK" + href: "https://www.swt-wc.usace.army.mil/TENK.lakepage.html" + - location_id: "THUN" + href: "https://www.swt-wc.usace.army.mil/THUN.lakepage.html" + - location_id: "TOMS" + href: "https://www.swt-wc.usace.army.mil/TOMS.lakepage.html" + - location_id: "TORO" + href: "https://www.swt-wc.usace.army.mil/TORO.lakepage.html" + - location_id: "WAUR" + href: "https://www.swt-wc.usace.army.mil/WAUR.lakepage.html" + - location_id: "WIST" + href: "https://www.swt-wc.usace.army.mil/WIST.lakepage.html" + +header: + project: + text: "Reservoir" # text in the far-left header cell + classes: "" # optional extra classes + align: center # optional: left|center|right + rows: + # Header row 1 + - - + text: "Pool Elevation" + colspan: 2 + - text: "Pool Limits" + colspan: 2 + - text: "Pool Occupied Storage" + colspan: 3 + - text: "8AM Status (cfs)" + colspan: 3 + - text: "Previous 24Hr Average (cfs)" + colspan: 3 + - text: "Precip (in)" + align: center + # Header row 2 + - - text: "" + rowspan: 2 + - text: "8AM
Current" + rowspan: 2 + - text: "8AM
Prev" + rowspan: 2 + - text: "Top Of" + colspan: 2 + - text: "(%)" + rowspan: 2 + - text: "C/F" + rowspan: 2 + - text: "(ac-ft)" + rowspan: 2 + - text: "Total Release" + rowspan: 2 + - text: "Power Release" + rowspan: 2 + - text: "Inflow" + rowspan: 2 + - text: "Total Release" + rowspan: 2 + - text: "Power Release" + rowspan: 2 + - text: "Inflow" + rowspan: 2 + - text: "24Hr" + rowspan: 2 + # Header row 3 + - - text: "Pool Elev (ft)" + - text: "Tailwater (ft)" + +columns: + # Timeseries at a specific time + - title: "Pool Elev (ft)" + key: "elev_ts" + office: "SWT" + unit: "EN" + precision: 2 + tsid: "{project}.Elev.Inst.1Hour.0.Ccp-Rev" + # override the global target time for THIS column (optional): + begin: "today 0800" + end: "today 0800" + # per-column render strings (optional): + missing: "--" + undefined: "~~~~.~~" + href: "https://water.usace.army.mil/office/{office}/ts?name={tsid}" + - title: "Pool Elev (ft)" + key: "elev_ts" + office: "SWT" + unit: "EN" + precision: 2 + tsid: "{project}.Elev.Inst.1Hour.0.Ccp-Rev" + begin: "yesterday 0800 America/Chicago" + end: "yesterday 0800 America/Chicago" + # override the global target time for THIS column (optional): + target_time: "2025-09-21T08:00:00-05:00" + # per-column render strings (optional): + missing: "--" + undefined: "~~~~.~~" + href: "https://water.usace.army.mil/office/{office}/ts?name={tsid}" + # Level identifier (no sampling window; we just read the level) + - title: "Top of Conservation (ft)" + key: "top_cons" + office: "SWT" + unit: "ft" + precision: 2 + level: "{project}.Elev.Inst.0.Top of Conservation" + # optional per column overrides + undefined: "n/a"