Improved support for datasets with time bands and non-0 center long

Author: jtroberts

README.md

@@ -69,6 +69,7 @@ coverage-vizgen uses a YAML dataset configuration file to specify settings for p
 * `s3_prefix` - S3 URI to upload MRFs to (if specified)
 * `idx_dir` - OnEarth idx directory to move MRF idx files to (if specified)
 * `is360` - Convert dataset from 0-360 bounds to -180 - 180 if true
+* `center_long` - The center longitude when convert from 0-360 (default: 0)
 * `speed_vars` - List of `u` and `v` variables to use for converting to speed
 
 ### Example Configurations

bin/vizgen.py

@@ -23,6 +23,7 @@
 import string
 import boto3
 import zarr
+import xarray as xr
 from botocore.exceptions import ClientError, NoCredentialsError
 from pathlib import Path
 from datetime import datetime
@@ -260,6 +261,14 @@ def nc2tiff(input_file, config, process_existing=False, limit=-1, create_cog=Fal
             config.reproject = ''
 
         counter += 1
+        if len(config.time_bands) <= 1:
+            # check for internal time dimension
+            dataset = xr.open_dataset(input_file)
+            time_dimension = dataset['time']
+            if len(time_dimension.values) > 1:
+                config.time_bands = [datetime.strptime(str(item)[:-4], '%Y-%m-%dT%H:%M:%S.%f') for item in time_dimension.values]
+            dataset.close()
+        print('Using time bands: ' + str(config.time_bands))
         for band, time in enumerate(config.time_bands):
             if time != '':
                 bandstring = '_b' + str(band+1)
@@ -296,7 +305,8 @@ def nc2tiff(input_file, config, process_existing=False, limit=-1, create_cog=Fal
                         output_file_360 = output_file.replace('.tiff', '_360.tiff')
                         print('Creating GeoTIFF file ' + output_file_360)
                         gdal_warp = ['gdalwarp', '-t_srs', 'WGS84', '-te', '-180', '-90', '180', '90', output_file,
-                                     output_file_360, '-wo', 'SOURCE_EXTRA=1000', '--config', 'CENTER_LONG', '0']
+                                     output_file_360, '-wo', 'SOURCE_EXTRA=1000', '--config',
+                                     'CENTER_LONG', str(config.center_long)]
                         print(gdal_warp)
                         process = subprocess.Popen(gdal_warp, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                         process.wait()
@@ -400,12 +410,20 @@ def create_mrf(input_files, config, layer_name, colormap, empty_tile):
                     else:
                         raise
             date_of_data = date.strftime('%Y%m%d')
-            print('Detected date: ' + date_of_data)
             time_of_data = date.strftime('%H%M%S')
             if config.time_bands != [''] and '_b' in filename:
                 band = int(filename.split('_b')[1].split('_')[0]) - 1
                 addtime = config.time_bands[band]
-                time_of_data = (date + timedelta(seconds=addtime)).strftime('%H%M%S')
+                # determine if sub-daily
+                if len(config.time_bands) > 1:
+                    if not str(config.time_bands[0]).isnumeric():
+                        date_of_data = config.time_bands[band].strftime('%Y%m%d')
+                        time_of_data = config.time_bands[band].strftime('%H%M%S')
+                    elif config.time_bands[1] - config.time_bands[0] > 366:
+                        time_of_data = (date + timedelta(seconds=addtime)).strftime('%H%M%S')
+                    else:
+                        date_of_data = (date + timedelta(days=addtime-1)).strftime('%Y%m%d')
+            print('Detected date: ' + date_of_data)
             output_dir = config.output_dir + '/' + layer_name + '/' + str(date.year)
             if not os.path.exists(output_dir):
                 os.makedirs(output_dir)

bin/vizingest.py

@@ -61,6 +61,7 @@ def __init__(self, config):
         self.s3_prefix = layer_config.get('s3_prefix')
         self.idx_dir = layer_config.get('idx_dir')
         self.is360 = layer_config.get('is360', False)
+        self.center_long = layer_config.get('center_long', 0)
         self.speed_vars = layer_config.get('speed_vars')
 
     def __str__(self):

requirements.txt

@@ -14,3 +14,6 @@ Rtree==1.0.1
 mapbox-vector-tile==2.0.1
 urllib3==1.26.16
 zarr==2.16.1
+xarray==2023.10.1
+h5netcdf==1.3.0
+netcdf4==1.6.5

version.sh

@@ -1,4 +1,4 @@
 #!/bin/sh
 
 # The VIZGEN version to use in the Docker image tag (e.g. coverage/vizgen:{VIZGEN_VERSION}).
-export VIZGEN_VERSION=0.6.2
+export VIZGEN_VERSION=0.7.0