BAGv2 verwerken van mutaties

bagv2/etl/conf/adres.cfg

@@ -20,7 +20,14 @@ chains = input_sql_address_tables|output_postgres_sql
 # Pre SQL file inputs to be executed
 [input_sql_address_tables]
 class = stetl.inputs.fileinput.StringFileInput
-file_path = sql/adres/adres-tabel.sql
+file_path = sql/fill-reltables.sql,
+            sql/adres/adres-tabel-plus-1.sql,
+            sql/adres/adres-tabel-plus-2.sql,
+            sql/adres/adres-tabel-plus-3.sql,
+            sql/adres/adres-tabel-plus-4.sql,
+            sql/adres/adres-tabel-plus-5.sql,
+            sql/adres/adres-tabel-totaal.sql,
+            sql/adres/adres-plus2adres-full-tabel.sql
 format_args = schema:{schema}
 
 #

bagv2/etl/conf/etl-bagv2-mutations.cfg

@@ -0,0 +1,113 @@
+# Stetl-Process-chains for extracting BAG VERSION 2 source data from input zip files to PostGIS.
+# A Chain is a series of Components: one Input, zero or more Filters and one Output.
+# The output of a Component is connected to the input of the next Component (except for
+# the final Output Component, which writes to the final destination, e.g. PostGIS or GeoPackage.
+# Stetl is the ETL tool used. No custom Python code is needed!
+#
+# Currently the following chains are executed in the following order:
+# - SQL pre:  DB initialization, create schema and tables
+# - Process "Leveringsdoc" put meta info like dataset date in nlx_bag_info table
+# - Process CBS gemeente provincie koppelingen
+# - BAG Woonplaats-Gemeente Koppeling (onderdeel van BAG Extract Kadaster product)
+# - input_bag_zip_file: inlezen BAG in PostGIS met LVBAG Driver
+# - input_sql_post: post-processing o.a. aanmaken indexen en VIEWs
+#
+# Just van den Broecke - 2021
+
+[etl]
+chains = input_sql_pre|output_postgres_sql,
+         input_bag_zipfile|output_bag_processor,
+         input_string_file|command_executor,
+         input_cbs_gemeenten_csv|output_postgres_gemeenten,
+         input_sql_post|output_postgres_sql
+
+
+#
+# INPUTS
+#
+
+# Pre SQL file inputs to be executed
+[input_sql_pre]
+class = stetl.inputs.fileinput.StringFileInput
+file_path = sql/mutations/truncate-tables.sql
+format_args = schema:{schema}
+
+# Post SQL file inputs to be executed
+[input_sql_post]
+class = stetl.inputs.fileinput.StringFileInput
+file_path = sql/mutations/apply-mutations.sql
+format_args = schema:{schema}
+
+[input_string_file]
+class = stetl.inputs.fileinput.StringFileInput
+file_path = data/cbs/gemeentelijke-indeling_command.txt
+
+[command_executor]
+class = stetl.filters.execfilter.CommandExecFilter
+
+# Input file gemeente-provincies van CBS per jaar
+[input_cbs_gemeenten_csv]
+class = stetl.inputs.fileinput.CsvFileInput
+file_path = data/cbs/gemeentelijke-indeling.csv
+delimiter = ;
+empty_string_is_none = True
+
+# BAG Extract LV zipfile.
+# Contains the zip files for each BAG obj type
+# Any substitutable values are specified in curly brackets e.g. {password}.
+# Actual values can be passed as args to `stetl`.
+
+[input_bag_zipfile]
+class = stetl.inputs.baginput.BAGInput
+file_path = {bag_mutations_input_file}
+temp_dir = temp/temp_dir
+multiprocessing = True
+
+
+#
+# OUTPUTS
+#
+
+# Algemene Postgres SQL Executor
+[output_postgres_sql]
+class = stetl.outputs.dboutput.PostgresDbOutput
+database = {pg_db}
+host = {pg_host}
+port = {pg_port}
+user = {pg_user}
+password = {password}
+schema = {schema}
+
+# Postgres provincie_gemeente tabel wegschrijven
+[output_postgres_gemeenten]
+class = stetl.outputs.dboutput.PostgresInsertOutput
+database = {pg_db}
+host = {pg_host}
+port = {pg_port}
+user = {pg_user}
+password = {password}
+schema = {schema}
+table = provincie_gemeente
+truncate = False
+
+# Process BAG 2.0 Extract mutation files
+[output_bag_processor]
+class = stetl.outputs.bagmutoutput.BAGMutOutput
+database = {pg_db}
+host = {pg_host}
+port = {pg_port}
+user = {pg_user}
+password = {password}
+schema = {schema}
+truncate = False
+temp_dir = temp/temp_dir
+process_lig = True
+process_num = True
+process_opr = True
+process_pnd = True
+process_sta = True
+process_vbo = True
+process_wpl = True
+process_gwr = True
+process_levering = True
+multiprocessing = True

bagv2/etl/conf/etl-imbag-2.1.0.cfg

@@ -26,12 +26,19 @@
 
 [etl]
 # [GDAL_3462]
+#chains = input_sql_pre|output_postgres_sql,
+#         input_bag_leveringsdoc_file|convert_vsi_to_etree_doc|transformer_xslt_leveringsdoc|output_ogr_postgis,
+#         input_bag_leveringsdoc_zipfile|extract_bag_leveringsdoc_file|convert_leveringsdoc_to_record|output_postgres_levering,
+#         input_string_file|command_executor,
+#         input_cbs_gemeenten_csv|output_postgres_gemeenten,
+#         input_bag_wpl_gem_koppeling_file|convert_vsi_to_etree_doc|transformer_xslt_gem_wpl|output_ogr_postgis,
+#         input_bag_zip_file|extract_bag_vsizip_file|expand_zip_archive|output_ogr2ogr,
+#         input_sql_post|output_postgres_sql
+
 chains = input_sql_pre|output_postgres_sql,
-         input_bag_leveringsdoc_file|convert_vsi_to_etree_doc|transformer_xslt_leveringsdoc|output_ogr_postgis,
+         input_bag_zipfile|output_bag_processor,
          input_string_file|command_executor,
          input_cbs_gemeenten_csv|output_postgres_gemeenten,
-         input_bag_wpl_gem_koppeling_file|convert_vsi_to_etree_doc|transformer_xslt_gem_wpl|output_ogr_postgis,
-         input_bag_zip_file|extract_bag_vsizip_file|expand_zip_archive|output_ogr2ogr,
          input_sql_post|output_postgres_sql
 
 
@@ -48,7 +55,7 @@ format_args = schema:{schema}
 # Post SQL file inputs to be executed
 [input_sql_post]
 class = stetl.inputs.fileinput.StringFileInput
-file_path = sql/finalize-tables.sql,sql/fill-reltables.sql,sql/create-views.sql,sql/create-indexes.sql
+file_path = sql/fill-reltables.sql,sql/create-views.sql,sql/create-indexes.sql,sql/bag-wpl-osm.sql
 format_args = schema:{schema}
 
 [input_string_file]
@@ -75,6 +82,16 @@ class = stetl.inputs.fileinput.VsiZipFileInput
 file_path = {bag_input_file}
 name_filter = {bag_obj_file_pat}{bag_file_wildcard}.{bag_file_ext}
 
+#[input_bag_zipfile]
+#class = stetl.inputs.fileinput.ZipFileInput
+#file_path = {bag_input_file}
+
+[input_bag_zipfile]
+class = stetl.inputs.baginput.BAGInput
+file_path = {bag_input_file}
+temp_dir = temp/temp_dir
+multiprocessing = True
+
 # Output GEM-WPL-RELATIE XML Input file as GDAL VSI string.
 [input_bag_wpl_gem_koppeling_file]
 class = stetl.inputs.fileinput.VsiZipFileInput
@@ -87,6 +104,11 @@ class = stetl.inputs.fileinput.VsiZipFileInput
 file_path = {bag_input_file}
 name_filter = Leveringsdocument-BAG-Extract.xml
 
+[input_bag_leveringsdoc_zipfile]
+class = stetl.inputs.fileinput.ZipFileInput
+file_path = {bag_input_file}
+name_filter = Leveringsdocument-BAG-Extract.xml
+
 
 #
 # FILTERS
@@ -98,6 +120,10 @@ class = stetl.filters.fileextractor.VsiFileExtractor
 file_path = temp/bagger.{bag_file_ext}
 delete_file = True
 
+[extract_bag_leveringsdoc_file]
+class = stetl.filters.fileextractor.ZipFileExtractor
+file_path = temp/Leveringsdocument-BAG-Extract.xml
+
 # Filter to expand a ZIP archive to a configured target dir
 [expand_zip_archive]
 class = stetl.filters.archiveexpander.ZipArchiveExpander
@@ -111,6 +137,10 @@ class = stetl.filters.formatconverter.FormatConverter
 input_format = gdal_vsi_path
 output_format = etree_doc
 
+# convert leverings document content to record
+[convert_leveringsdoc_to_record]
+class = stetl.filters.bagfilter.LeveringFilter
+
 # XSLT transformer GEM-WPL-RELATIE to OGR Simple Feature GML doctree
 [transformer_xslt_gem_wpl]
 class = stetl.filters.xsltfilter.XsltFilter
@@ -146,6 +176,41 @@ password = {password}
 schema = {schema}
 table = provincie_gemeente
 
+# Insert levering XML into nlx_bag_info table
+[output_postgres_levering]
+class = stetl.outputs.dboutput.PostgresInsertOutput
+database = {pg_db}
+host = {pg_host}
+port = {pg_port}
+user = {pg_user}
+password = {password}
+schema = {schema}
+table = nlx_bag_info
+key = sleutel
+replace = true
+
+# Process BAG 2.0 Extract files
+[output_bag_processor]
+class = stetl.outputs.bagoutput.BAGOutput
+database = {pg_db}
+host = {pg_host}
+port = {pg_port}
+user = {pg_user}
+password = {password}
+schema = {schema}
+#truncate = True
+temp_dir = temp/temp_dir
+process_lig = True
+process_num = True
+process_opr = True
+process_pnd = True
+process_sta = True
+process_vbo = True
+process_wpl = True
+process_gwr = True
+process_levering = True
+multiprocessing = True
+
 # The ogr2ogr command-line for generic GML Features.
 [output_ogr_postgis]
 class = stetl.outputs.ogroutput.Ogr2OgrOutput

bagv2/etl/mut-etl.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+# ETL voor BAG Extract versie 2 XML met gebruik Stetl en GDAL LVBAG Driver.
+#
+# Dit is een front-end/wrapper shell-script om uiteindelijk Stetl met een configuratie
+# (etl-imbag.cfg) en parameters (options/myoptions.args) aan te roepen.
+#
+# Author: Just van den Broecke
+#
+
+NLX_ETL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# Uiteindelijke commando. Kan ook gewoon "stetl -c conf/etl-bagv2-mutations.cfg -a ..." worden indien Stetl installed
+${NLX_ETL_DIR}/stetl.sh -c conf/etl-bagv2-mutations.cfg $@

bagv2/etl/options/common.args

@@ -9,6 +9,12 @@
 # 9999PND15022021.zip
 bag_obj_file_pat=[0-9][0-9][0-9][0-9][LSVNPOW][ITBUNPP][GAOMDRL][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]
 
+# Typical BAG mutation file prefix pattern, .zip or XMLs
+# Examples:
+# 9999IOMUT20042022-21042022.zip
+# 9999MUT20042022-21042022.zip
+bag_mut_file_pat=9999[IO]*?MUT[0-9]{8}\-[0-9]{8}
+
 # See ETL config file.
 # [GDAL_3462]
 # bag_obj_file_pat_no_wpl=[0-9][0-9][0-9][0-9][LSVNPO][ITBUNP][GAOMDR][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]

bagv2/etl/options/default.args

@@ -4,6 +4,7 @@
 
 # INPUT: gml files, point to directory or file(s) pattern
 bag_input_file=test/data/lv/BAGNLDL-15092020-small.zip
+bag_mutations_input_file=test/data/lv/BAGNLDM-08042022-09042022.zip
 bag_file_ext=zip
 
 # OUTPUT: PostGIS settings