From 482da5397652076408ec9f15a6690736c2332717 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Wed, 10 Sep 2025 16:47:07 -0400 Subject: [PATCH 1/6] Structure docs --- docs/datacube-benchmark/api.md | 18 +-------- docs/datacube-benchmark/titiler.md | 15 ++++++++ docs/index.md | 20 +++++----- .../overview.md} | 0 .../titiler-cmr/titiler-cmr-benchmark.ipynb | 0 .../titiler-cmr-statistics-benchmark.ipynb | 0 .../titiler}/titiler-ecosystem.md | 0 .../bloated-datatypes.ipynb | 0 .../dispersed-metadata.ipynb | 0 .../fsspec-caching-defaults.md | 0 docs/{ => worst-practices}/gdal-defaults.md | 0 .../massive-chunks.ipynb | 0 .../non-standardized-metadata.ipynb | 0 docs/{ => worst-practices}/old-libraries.md | 0 docs/{ => worst-practices}/tiny-chunks.ipynb | 0 .../tiny-coordinate-chunks.ipynb | 0 .../xarray-combine-defaults.md | 0 mkdocs.yml | 37 ++++++++++--------- 18 files changed, 45 insertions(+), 45 deletions(-) create mode 100644 docs/datacube-benchmark/titiler.md rename docs/{explanation/visualization.md => visualization/overview.md} (100%) rename docs/{ => visualization/titiler}/titiler-cmr/titiler-cmr-benchmark.ipynb (100%) rename docs/{ => visualization/titiler}/titiler-cmr/titiler-cmr-statistics-benchmark.ipynb (100%) rename docs/{explanation => visualization/titiler}/titiler-ecosystem.md (100%) rename docs/{ => worst-practices}/bloated-datatypes.ipynb (100%) rename docs/{ => worst-practices}/dispersed-metadata.ipynb (100%) rename docs/{ => worst-practices}/fsspec-caching-defaults.md (100%) rename docs/{ => worst-practices}/gdal-defaults.md (100%) rename docs/{ => worst-practices}/massive-chunks.ipynb (100%) rename docs/{ => worst-practices}/non-standardized-metadata.ipynb (100%) rename docs/{ => worst-practices}/old-libraries.md (100%) rename docs/{ => worst-practices}/tiny-chunks.ipynb (100%) rename docs/{ => worst-practices}/tiny-coordinate-chunks.ipynb (100%) rename docs/{ => worst-practices}/xarray-combine-defaults.md (100%) diff --git a/docs/datacube-benchmark/api.md b/docs/datacube-benchmark/api.md index a411237..2ac29e3 100644 --- a/docs/datacube-benchmark/api.md +++ b/docs/datacube-benchmark/api.md @@ -1,4 +1,4 @@ -# API Documentation +# Datacube access ::: datacube_benchmark.utils.array_storage_size @@ -16,20 +16,4 @@ ::: datacube_benchmark.benchmark_statistics -::: datacube_benchmark.tiling_benchmark_summary - -::: datacube_benchmark.TiTilerCMRBenchmarker - -::: datacube_benchmark.tiling.DatasetParams - ::: datacube_benchmark.types.TARGET_SHAPES - -## General Tiling Utilities - -::: datacube_benchmark.tiling.get_surrounding_tiles - -::: datacube_benchmark.tiling.fetch_tile - -::: datacube_benchmark.tiling.get_tileset_tiles - -::: datacube_benchmark.tiling.create_bbox_feature \ No newline at end of file diff --git a/docs/datacube-benchmark/titiler.md b/docs/datacube-benchmark/titiler.md new file mode 100644 index 0000000..6cff2d9 --- /dev/null +++ b/docs/datacube-benchmark/titiler.md @@ -0,0 +1,15 @@ +# Datacube tiling + +::: datacube_benchmark.tiling_benchmark_summary + +::: datacube_benchmark.TiTilerCMRBenchmarker + +::: datacube_benchmark.tiling.DatasetParams + +::: datacube_benchmark.tiling.get_surrounding_tiles + +::: datacube_benchmark.tiling.fetch_tile + +::: datacube_benchmark.tiling.get_tileset_tiles + +::: datacube_benchmark.tiling.create_bbox_feature \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 4c8fade..b8ee217 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,39 +10,39 @@ The datacube worst practices guidebook is intended as an open, community-maintai --- - [:octicons-arrow-right-24: Tiny data chunks](tiny-chunks.ipynb) + [:octicons-arrow-right-24: Tiny data chunks](worst-practices/tiny-chunks.ipynb) - __Massive data chunks__ --- - [:octicons-arrow-right-24: Massive data chunks](massive-chunks.ipynb) + [:octicons-arrow-right-24: Massive data chunks](worst-practices/massive-chunks.ipynb) - __Tiny coordinate chunks__ --- - [:octicons-arrow-right-24: Tiny coordinate chunks](tiny-coordinate-chunks.ipynb) + [:octicons-arrow-right-24: Tiny coordinate chunks](worst-practices/tiny-coordinate-chunks.ipynb) - __Dispersed metadata__ --- - [:octicons-arrow-right-24: Dispersed metadata](dispersed-metadata.ipynb) + [:octicons-arrow-right-24: Dispersed metadata](worst-practices/dispersed-metadata.ipynb) - __Non-standardized metadata__ --- - [:octicons-arrow-right-24: Non-standardized metadata](non-standardized-metadata.ipynb) + [:octicons-arrow-right-24: Non-standardized metadata](worst-practices/non-standardized-metadata.ipynb) - __Bloated datatypes__ --- - [:octicons-arrow-right-24: Bloated datatypes](bloated-datatypes.ipynb) + [:octicons-arrow-right-24: Bloated datatypes](worst-practices/bloated-datatypes.ipynb) @@ -55,25 +55,25 @@ The datacube worst practices guidebook is intended as an open, community-maintai --- - [:octicons-arrow-right-24: Default FSSpec Caching](fsspec-caching-defaults.md) + [:octicons-arrow-right-24: Default FSSpec Caching](worst-practices/fsspec-caching-defaults.md) - __Default GDAL Config__ --- - [:octicons-arrow-right-24: Default GDAL Config](gdal-defaults.md) + [:octicons-arrow-right-24: Default GDAL Config](worst-practices/gdal-defaults.md) - __Default Xarray combine arguments__ --- - [:octicons-arrow-right-24: Default Xarray combine arguments](xarray-combine-defaults.md) + [:octicons-arrow-right-24: Default Xarray combine arguments](worst-practices/xarray-combine-defaults.md) - __Using old libraries__ --- - [:octicons-arrow-right-24: Using old libraries](old-libraries.md) + [:octicons-arrow-right-24: Using old libraries](worst-practices/old-libraries.md) diff --git a/docs/explanation/visualization.md b/docs/visualization/overview.md similarity index 100% rename from docs/explanation/visualization.md rename to docs/visualization/overview.md diff --git a/docs/titiler-cmr/titiler-cmr-benchmark.ipynb b/docs/visualization/titiler/titiler-cmr/titiler-cmr-benchmark.ipynb similarity index 100% rename from docs/titiler-cmr/titiler-cmr-benchmark.ipynb rename to docs/visualization/titiler/titiler-cmr/titiler-cmr-benchmark.ipynb diff --git a/docs/titiler-cmr/titiler-cmr-statistics-benchmark.ipynb b/docs/visualization/titiler/titiler-cmr/titiler-cmr-statistics-benchmark.ipynb similarity index 100% rename from docs/titiler-cmr/titiler-cmr-statistics-benchmark.ipynb rename to docs/visualization/titiler/titiler-cmr/titiler-cmr-statistics-benchmark.ipynb diff --git a/docs/explanation/titiler-ecosystem.md b/docs/visualization/titiler/titiler-ecosystem.md similarity index 100% rename from docs/explanation/titiler-ecosystem.md rename to docs/visualization/titiler/titiler-ecosystem.md diff --git a/docs/bloated-datatypes.ipynb b/docs/worst-practices/bloated-datatypes.ipynb similarity index 100% rename from docs/bloated-datatypes.ipynb rename to docs/worst-practices/bloated-datatypes.ipynb diff --git a/docs/dispersed-metadata.ipynb b/docs/worst-practices/dispersed-metadata.ipynb similarity index 100% rename from docs/dispersed-metadata.ipynb rename to docs/worst-practices/dispersed-metadata.ipynb diff --git a/docs/fsspec-caching-defaults.md b/docs/worst-practices/fsspec-caching-defaults.md similarity index 100% rename from docs/fsspec-caching-defaults.md rename to docs/worst-practices/fsspec-caching-defaults.md diff --git a/docs/gdal-defaults.md b/docs/worst-practices/gdal-defaults.md similarity index 100% rename from docs/gdal-defaults.md rename to docs/worst-practices/gdal-defaults.md diff --git a/docs/massive-chunks.ipynb b/docs/worst-practices/massive-chunks.ipynb similarity index 100% rename from docs/massive-chunks.ipynb rename to docs/worst-practices/massive-chunks.ipynb diff --git a/docs/non-standardized-metadata.ipynb b/docs/worst-practices/non-standardized-metadata.ipynb similarity index 100% rename from docs/non-standardized-metadata.ipynb rename to docs/worst-practices/non-standardized-metadata.ipynb diff --git a/docs/old-libraries.md b/docs/worst-practices/old-libraries.md similarity index 100% rename from docs/old-libraries.md rename to docs/worst-practices/old-libraries.md diff --git a/docs/tiny-chunks.ipynb b/docs/worst-practices/tiny-chunks.ipynb similarity index 100% rename from docs/tiny-chunks.ipynb rename to docs/worst-practices/tiny-chunks.ipynb diff --git a/docs/tiny-coordinate-chunks.ipynb b/docs/worst-practices/tiny-coordinate-chunks.ipynb similarity index 100% rename from docs/tiny-coordinate-chunks.ipynb rename to docs/worst-practices/tiny-coordinate-chunks.ipynb diff --git a/docs/xarray-combine-defaults.md b/docs/worst-practices/xarray-combine-defaults.md similarity index 100% rename from docs/xarray-combine-defaults.md rename to docs/worst-practices/xarray-combine-defaults.md diff --git a/mkdocs.yml b/mkdocs.yml index 4f734ca..e01ccfd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,28 +14,29 @@ nav: - "index.md" - Datacube Worst Practices: - Common production gotchas: - - Tiny data chunks: "tiny-chunks.ipynb" - - Massive data chunks: "massive-chunks.ipynb" - - Tiny coordinate chunks: "tiny-coordinate-chunks.ipynb" - - Dispersed metadata: "dispersed-metadata.ipynb" - - Non-standardized metadata: "non-standardized-metadata.ipynb" - - Bloated datatypes: "bloated-datatypes.ipynb" + - Tiny data chunks: "worst-practices/tiny-chunks.ipynb" + - Massive data chunks: "worst-practices/massive-chunks.ipynb" + - Tiny coordinate chunks: "worst-practices/tiny-coordinate-chunks.ipynb" + - Dispersed metadata: "worst-practices/dispersed-metadata.ipynb" + - Non-standardized metadata: "worst-practices/non-standardized-metadata.ipynb" + - Bloated datatypes: "worst-practices/bloated-datatypes.ipynb" - Common usage gotchas: - - Default Xarray combine arguments: "xarray-combine-defaults.md" - - Default FSSpec caching arguments: "fsspec-caching-defaults.md" - - Default GDAL config: "gdal-defaults.md" - - Using old libraries: "old-libraries.md" - + - Default Xarray combine arguments: "worst-practices/xarray-combine-defaults.md" + - Default FSSpec caching arguments: "worst-practices/fsspec-caching-defaults.md" + - Default GDAL config: "worst-practices/gdal-defaults.md" + - Using old libraries: "worst-practices/old-libraries.md" - Datacube Visualization: - - "explanation/visualization.md" - - "explanation/titiler-ecosystem.md" - - - Titiler CMR Benchmarking: - - Tile Generation Performance: "titiler-cmr/titiler-cmr-benchmark.ipynb" - - Statistics Endpoint Performance: "titiler-cmr/titiler-cmr-statistics-benchmark.ipynb" + - Overview: "visualization/overview.md" + - Titiler: + - Titiler ecosystem overview: "visualization/titiler/titiler-ecosystem.md" + - Performance benchmarking: + - Titiler-CMR tile generation: "visualization/titiler/titiler-cmr/titiler-cmr-benchmark.ipynb" + - Titiler-CMR timeseries statistics: "visualization/titiler/titiler-cmr/titiler-cmr-statistics-benchmark.ipynb" - - Reference: + - API Reference: - "datacube-benchmark/api.md" + - "datacube-benchmark/titiler.md" + watch: - packages - docs From a5775e915f1a5b862c756f5479a8c35dc28d9ba1 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 11 Sep 2025 10:52:33 -0400 Subject: [PATCH 2/6] Rename --- docs/visualization/titiler/{titiler-ecosystem.md => overview.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/visualization/titiler/{titiler-ecosystem.md => overview.md} (100%) diff --git a/docs/visualization/titiler/titiler-ecosystem.md b/docs/visualization/titiler/overview.md similarity index 100% rename from docs/visualization/titiler/titiler-ecosystem.md rename to docs/visualization/titiler/overview.md From ed6640dd6f0b5bf6f15ee44a3fce45e036dce600 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 11 Sep 2025 11:15:31 -0400 Subject: [PATCH 3/6] Update docs structure --- .github/workflows/docs.yml | 55 +++++++++++++++++ .../datacube-benchmark.md} | 0 .../titiler-benchmark.md} | 2 +- docs/visualization/titiler/overview.md | 61 ++++++++----------- mkdocs.yml | 7 ++- 5 files changed, 86 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/docs.yml rename docs/{datacube-benchmark/api.md => api-reference/datacube-benchmark.md} (100%) rename docs/{datacube-benchmark/titiler.md => api-reference/titiler-benchmark.md} (94%) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..5e12e05 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,55 @@ +name: Deploy to GitHub Pages + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 1 + + - name: Install a specific version of uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + version: "0.5.x" + + - name: Install dependencies + run: uv sync + + - name: Build docs + run: uv run -- mkdocs build --strict + + - name: Upload artifact + id: deployment + uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 + with: + path: ./site + retention-days: 1 + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 + timeout-minutes: 10 diff --git a/docs/datacube-benchmark/api.md b/docs/api-reference/datacube-benchmark.md similarity index 100% rename from docs/datacube-benchmark/api.md rename to docs/api-reference/datacube-benchmark.md diff --git a/docs/datacube-benchmark/titiler.md b/docs/api-reference/titiler-benchmark.md similarity index 94% rename from docs/datacube-benchmark/titiler.md rename to docs/api-reference/titiler-benchmark.md index 6cff2d9..6fe21cf 100644 --- a/docs/datacube-benchmark/titiler.md +++ b/docs/api-reference/titiler-benchmark.md @@ -1,4 +1,4 @@ -# Datacube tiling +# Dynamic tiling ::: datacube_benchmark.tiling_benchmark_summary diff --git a/docs/visualization/titiler/overview.md b/docs/visualization/titiler/overview.md index 64e45c7..2ff994b 100644 --- a/docs/visualization/titiler/overview.md +++ b/docs/visualization/titiler/overview.md @@ -1,52 +1,43 @@ -# Components of the TiTiler Ecosystem +# Titiler Ecosystem Overview -The TiTiler ecosystem is a comprehensive suite of Python tools for creating dynamic tile servers from geospatial datasets. The components are organized by their primary function: +The TiTiler ecosystem is a comprehensive suite of Python tools for creating dynamic tile servers from geospatial datasets. The components are organized by their primary function and built upon a layered architecture that provides flexibility and specialization for different use cases. -## Core TiTiler Framework +## Architecture and Relationships -**titiler.core** - Foundation libraries for creating FastAPI applications that serve dynamic tiles from Cloud Optimized GeoTIFFs (COGs) and SpatioTemporal Asset Catalog (STAC) items. +The TiTiler ecosystem follows a layered architecture that promotes code reuse and specialization: -**titiler.xarray** - Specialized extension for dynamically serving tiles from multi-dimensional datasets stored in Zarr or NetCDF formats. +### Foundation Layer +- **[rio-tiler](https://github.com/cogeotiff/rio-tiler)**: Core tile generation engine +- **[titiler.core](https://github.com/developmentseed/titiler/tree/main/src/titiler/core)**: Base FastAPI framework and patterns -**titiler.extensions** - Plugin system providing additional functionality for TiTiler factories, such as custom algorithms, authentication, and data processing extensions. +### Extension Layer +- **[titiler.xarray](https://github.com/developmentseed/titiler/tree/main/src/titiler/xarray)**: Multidimensional data support extending titiler.core +- **[titiler.extensions](https://github.com/developmentseed/titiler/tree/main/src/titiler/extensions)**: Plugin system for custom functionality +- **[titiler.mosaic](https://github.com/developmentseed/titiler/tree/main/src/titiler/mosaic)**: Multi-source tiling capabilities -**titiler.mosaic** - Extension for dynamically serving tiles from multiple sources using the MosaicJSON specification. +### Application Layer +- **[titiler.application](https://github.com/developmentseed/titiler/tree/main/src/titiler/application)**: Reference implementation using titiler.core +- **[titiler-multidim](https://github.com/developmentseed/titiler-multidim)**: Prototype application using titiler.xarray + optimizations +- **[titiler-cmr](https://github.com/developmentseed/titiler-cmr)**: NASA-specific application using titiler.core + CMR integration +- **[titiler-eopf](https://github.com/EOPF-Explorer/titiler-eopf)**: ESA-specific application using titiler.xarray + EOPF integration -**titiler.application** - Complete reference implementation demonstrating a FastAPI application with full support for COGs, STAC items, and MosaicJSON mosaics. +### Key Relationships -## Specialized Applications +#### **titiler.core → titiler.xarray** -**titiler-cmr** - NASA-focused application that accepts Concept IDs and uses the Common Metadata Repository (CMR) to discover and serve associated granules as tiles. +`titiler.xarray` extends the core framework with xarray-based readers and multidimensional dataset support, inheriting all core functionality while adding temporal and dimensional processing capabilities. -**titiler-multidim** - Application specifically designed for multi-dimensional datasets, built on titiler.xarray for handling complex scientific data formats. +#### **titiler.xarray → titiler-multidim / titiler-eopf applications** -## Core Libraries +Both `titiler-multidim` and `titiler.eopf` are built on `titiler.xarray`, leveraging its multidimensional capabilities while adding their own optimizations: -**rio-tiler** - The foundational library that handles the core tile generation logic, dynamically creating map tiles from raster data sources including COGs. +- `titiler-multidim` adds Redis caching, VEDA platform integration, and prototypes of optimizations +- `titiler-eopf` adds EOPF-specific data structures, collection/item routing, and ESA workflow integrations -**rio-cogeo** - Command-line tool and library for creating and validating Cloud Optimized GeoTIFFs, ensuring optimal performance for tile serving. +#### **titiler.core → titiler-cmr application** -**rio-viz** - Lightweight visualization tool for locally exploring and debugging raster datasets during development. - -## Infrastructure Components - -**cogeo-mosaic** - Serverless infrastructure toolkit (designed for AWS Lambda) that implements the MosaicJSON specification for creating and serving mosaicked tile sets. - -## Development Tools - -**tilebench** - Performance analysis tool that measures how many HTTP requests are required to generate tiles from different data sources, helping optimize tile server configurations. - -## Extensions and Plugins - -**rio-tiler-mvt** - Plugin that extends rio-tiler to generate Mapbox Vector Tiles (MVT) from raster data sources, enabling vector-based visualizations. - -## Standards and Specifications - -**MosaicJSON** - Open specification for creating spatial indexes that efficiently link multiple COGs to XYZ tile coordinates, enabling seamless mosaicking of large datasets. - -## Legacy Components - -**riotiler_mosaic** - **(Deprecated)** - Former rio-tiler plugin for creating tiles from multiple observations. This functionality has been integrated directly into rio-tiler and cogeo-mosaic. +`titiler-cmr` is built directly on `titiler.core` rather than `titiler.xarray` due to the development timeline of the two projects. In the future, we anticipate +`titiler-cmr` will depend on `titiler.xarray`, with progress tracked by [titiler-cmr issue #35](https://github.com/developmentseed/titiler-cmr/issues/35) --- diff --git a/mkdocs.yml b/mkdocs.yml index e01ccfd..608efe9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,14 +28,15 @@ nav: - Datacube Visualization: - Overview: "visualization/overview.md" - Titiler: - - Titiler ecosystem overview: "visualization/titiler/titiler-ecosystem.md" + - Ecosystem overview: "visualization/titiler/overview.md" - Performance benchmarking: - Titiler-CMR tile generation: "visualization/titiler/titiler-cmr/titiler-cmr-benchmark.ipynb" - Titiler-CMR timeseries statistics: "visualization/titiler/titiler-cmr/titiler-cmr-statistics-benchmark.ipynb" - API Reference: - - "datacube-benchmark/api.md" - - "datacube-benchmark/titiler.md" + - Benchmarking: + - "api-reference/datacube-benchmark.md" + - "api-reference/titiler-benchmark.md" watch: - packages From b9b54ee6d1d35cb0bdb7a50bec5b27799f1ef576 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Thu, 11 Sep 2025 11:29:40 -0400 Subject: [PATCH 4/6] Add deployment APIs --- docs/api-reference/titiler/index.md | 61 +++++++++++++++++++ docs/api-reference/titiler/titiler-cmr.md | 37 +++++++++++ docs/api-reference/titiler/titiler-core.md | 40 ++++++++++++ .../api-reference/titiler/titiler-multidim.md | 39 ++++++++++++ mkdocs.yml | 5 ++ 5 files changed, 182 insertions(+) create mode 100644 docs/api-reference/titiler/index.md create mode 100644 docs/api-reference/titiler/titiler-cmr.md create mode 100644 docs/api-reference/titiler/titiler-core.md create mode 100644 docs/api-reference/titiler/titiler-multidim.md diff --git a/docs/api-reference/titiler/index.md b/docs/api-reference/titiler/index.md new file mode 100644 index 0000000..b96748f --- /dev/null +++ b/docs/api-reference/titiler/index.md @@ -0,0 +1,61 @@ +# TiTiler API Reference + +This section provides interactive API documentation for the TiTiler ecosystem components. Each application has its own specialized API while sharing common patterns from the core framework. + +## Available Applications + +