From bcf76214aeacc5da218816be1e767b83277ba045 Mon Sep 17 00:00:00 2001 From: Mohd Yaseen Date: Wed, 27 Mar 2024 14:20:25 +0530 Subject: [PATCH] ACE-2020-add-timestamp_ntz-in-databricks-conn-config --- .../targets/target-setup/databricks/databricks-aws.md | 9 ++++++++- .../target-setup/databricks/databricks-azure.md | 10 +++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/content/docs/targets/target-setup/databricks/databricks-aws.md b/content/docs/targets/target-setup/databricks/databricks-aws.md index 6033fb96..4581d64b 100644 --- a/content/docs/targets/target-setup/databricks/databricks-aws.md +++ b/content/docs/targets/target-setup/databricks/databricks-aws.md @@ -64,7 +64,14 @@ Replicant requires the Databricks JDBC Driver as a dependency. To obtain the app max-connections: 30 #Maximum number of connections Replicant can open in Databricks max-retries: 100 #Number of times any operation on the source system will be re-attempted on failures. retry-wait-duration-ms: 1000 #Duration in milliseconds replicant should wait before performing then next retry of a failed operation + + supports-timestamp-ntz: false #If disabled, uses timestamp data type instead of timestamp_ntz + date-format: #default is yyyy-MM-dd, specify the date format if source provides dates in a format other than default + timestamp-format: #default 'yyyy-MM-dd HH:mm:ss', specify the format if source provides timestamps in a format other than default ``` + - **`supports-timestamp-ntz`**: Specifies which datatype to use for timestamp data. `timestamp_ntz` stores timestamp without timezone. + - **`date-format`**: Specify the data format of the source database. + - **`timestamp-format`**: Specify the timestamp format of the source database. Replace the following: - *`HOSTNAME`*: the hostname of your Databricks host @@ -302,4 +309,4 @@ As of now, note the following about the state of Arcion's Unity Catalog support: *Default: `PARQUET`*. - We use `SparkJDBC42` driver for Legacy Databricks (`DATABRICKS_DELTALAKE`) and `DatabricksJDBC42` for Unity catalog (`DATABRICKS_LAKEHOUSE`). For instructions on how to obtain these drivers, see [Obtain the JDBC Driver for Databricks](#i-obtain-the-jdbc-driver-for-databricks). - Replicant supports Unity Catalog on AWS and Azure platforms. -- To configure [Mapper file]({{< ref "docs/targets/configuration-files/mapper-reference" >}}) in Unity Catalog, see [Mapping in Unity Catalog]({{< ref "docs/targets/configuration-files/mapper-reference#mapping-in-unity-catalog" >}}). \ No newline at end of file +- To configure [Mapper file]({{< ref "docs/targets/configuration-files/mapper-reference" >}}) in Unity Catalog, see [Mapping in Unity Catalog]({{< ref "docs/targets/configuration-files/mapper-reference#mapping-in-unity-catalog" >}}). diff --git a/content/docs/targets/target-setup/databricks/databricks-azure.md b/content/docs/targets/target-setup/databricks/databricks-azure.md index a8b52b9e..30e62764 100644 --- a/content/docs/targets/target-setup/databricks/databricks-azure.md +++ b/content/docs/targets/target-setup/databricks/databricks-azure.md @@ -134,7 +134,15 @@ In this step, you need to provide the Databricks connection details to Arcion. T host: "HOSTNAME" port: "PORT_NUMBER" max-connections: 30 + + supports-timestamp-ntz: false #If disabled, uses timestamp data type instead of timestamp_ntz + date-format: #default is yyyy-MM-dd, specify the date format if source provides dates in a format other than default + timestamp-format: #default 'yyyy-MM-dd HH:mm:ss', specify the format if source provides timestamps in a format other than default ``` + - **`supports-timestamp-ntz`**: Specifies which datatype to use for timestamp data. `timestamp_ntz` stores timestamp without timezone. + - **`date-format`**: Specify the data format of the source database. + - **`timestamp-format`**: Specify the timestamp format of the source database. + Replace the following: - *`JDBC_URL`*: the JDBC URL that you retrieve [from the connection details for a cluster](#get-connection-details-for-a-cluster) - *`HOSTNAME`*: the hostname of your Databricks host that you retrieved [from the connection details for a cluster](#get-connection-details-for-a-cluster) @@ -342,4 +350,4 @@ As of now, note the following about the state of Arcion's Unity Catalog support: *Default: `PARQUET`*. - We use `SparkJDBC42` driver for Legacy Databricks (`DATABRICKS_DELTALAKE`) and `DatabricksJDBC42` for Unity catalog (`DATABRICKS_LAKEHOUSE`). For instructions on how to obtain these drivers, see [Obtain the JDBC Driver for Databricks](#i-obtain-the-jdbc-driver-for-databricks). - Replicant supports Unity Catalog on AWS and Azure platforms. -- To configure [Mapper file]({{< ref "docs/targets/configuration-files/mapper-reference" >}}) in Unity Catalog, see [Mapping in Unity Catalog]({{< ref "docs/targets/configuration-files/mapper-reference#mapping-in-unity-catalog" >}}). \ No newline at end of file +- To configure [Mapper file]({{< ref "docs/targets/configuration-files/mapper-reference" >}}) in Unity Catalog, see [Mapping in Unity Catalog]({{< ref "docs/targets/configuration-files/mapper-reference#mapping-in-unity-catalog" >}}).