RELEASE v2.4.0 (#86)

Co-authored-by: Nguyen Ngoc Son <jopoly>

Author: jopoly

META.json

@@ -2,15 +2,15 @@
    "name": "sqlite_fdw",
    "abstract": "Foreign Data Wrapper for SQLite databases",
    "description": "PostgreSQL extension which implements a Foreign Data Wrapper (FDW) for SQLite databases.",
-   "version": "2.3.0",
+   "version": "2.4.0",
    "maintainer": "pgspider",
    "license": "postgresql",
    "provides": {
       "sqlite_fdw": {
          "abstract": "Foreign Data Wrapper for SQLite databases",
          "file": "sqlite_fdw.c",
          "docfile": "README.md",
-         "version": "2.3.0"
+         "version": "2.4.0"
       }
    },
    "prereqs": {

Makefile

@@ -37,8 +37,8 @@ include $(PGXS)
 ifndef MAJORVERSION
 MAJORVERSION := $(basename $(VERSION))
 endif
-ifeq (,$(findstring $(MAJORVERSION), 11 12 13 14 15))
-$(error PostgreSQL 11, 12, 13, 14 or 15 is required to compile this extension)
+ifeq (,$(findstring $(MAJORVERSION), 12 13 14 15 16))
+$(error PostgreSQL 12, 13, 14, 15 or 16 is required to compile this extension)
 endif
 
 else

README.md

@@ -2,7 +2,7 @@ SQLite Foreign Data Wrapper for PostgreSQL
 ==========================================
 
 This is a foreign data wrapper (FDW) to connect [PostgreSQL](https://www.postgresql.org/)
-to [SQLite](https://sqlite.org/) database file. This FDW works with PostgreSQL 11, 12, 13, 14, 15 and confirmed with SQLite 3.38.5.
+to [SQLite](https://sqlite.org/) database file. This FDW works with PostgreSQL 12, 13, 14, 15, 16 and confirmed with SQLite 3.42.0.
 
 <img src="https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg" align="center" height="100" alt="PostgreSQL"/>	+	<img src="https://upload.wikimedia.org/wikipedia/commons/3/38/SQLite370.svg" align="center" height="100" alt="SQLite"/>
 
@@ -93,7 +93,7 @@ For Debian or Ubuntu:
 `apt-get install libsqlite3-dev`
 `apt-get install postgresql-server-dev-XX`, where XX matches your postgres version, i.e. `apt-get install postgresql-server-dev-15`
 
-You can also [download SQLite source code][1] and [build SQLite][2].
+You can also [download SQLite source code][1] and [build SQLite][2] with FTS5 for full-text search.
 
 #### 2. Build and install sqlite_fdw
 
@@ -120,31 +120,31 @@ Usage
 - **database** as *string*, **required**
 
   SQLite database path.
-  
+
 - **updatable** as *boolean*, optional, default *true*
 
   This option allow or disallow write operations on SQLite database file.
-    
+
 - **truncatable** as *boolean*, optional, default *true*
 
   Allows foreign tables to be truncated using the `TRUNCATE` command.
-  
+
 - **keep_connections** as *boolean*, optional, default *true*
-  
+
   Allows to keep connections to SQLite while there is no SQL operations between PostgreSQL and SQLite.
-  
+
 - **batch_size** as *integer*, optional, default *1*
 
   Specifies the number of rows which should be inserted in a single `INSERT` operation. This setting can be overridden for individual tables.
-  
+
 ## CREATE USER MAPPING options
 
 There is no user or password conceptions in SQLite, hence `sqlite_fdw` no need any `CREATE USER MAPPING` command.
 
 In OS `sqlite_fdw` works as executed code with permissions of user of PostgreSQL server. Usually it is `postgres` OS user. For interacting with SQLite database without access errors ensure this user have follow permissions:
 - read permission on all directories by path to the SQLite database file;
 - read permission on SQLite database file;
-- write permissions both on SQLite database file and *directory it contains* if you need a modification. During `INSERT`, `UPDATE` or `DELETE` in SQLite database, SQLite engine functions makes temporary files with transaction data in the directory near SQLite database file. Hence without write permissions you'll have a message `failed to execute remote SQL: rc=8 attempt to write a readonly database`. 
+- write permissions both on SQLite database file and *directory it contains* if you need a modification. During `INSERT`, `UPDATE` or `DELETE` in SQLite database, SQLite engine functions makes temporary files with transaction data in the directory near SQLite database file. Hence without write permissions you'll have a message `failed to execute remote SQL: rc=8 attempt to write a readonly database`.
 
 ## CREATE FOREIGN TABLE options
 
@@ -156,17 +156,17 @@ In OS `sqlite_fdw` works as executed code with permissions of user of PostgreSQL
   SQLite table name. Use if not equal to name of foreign table in PostgreSQL. Also see about [identifier case handling](#identifier-case-handling).
 
 - **truncatable** as *boolean*, optional, default from the same `CREATE SERVER` option
-  
+
   See `CREATE SERVER` options section for details.
 
 - **batch_size** as *integer*, optional, default from the same `CREATE SERVER` option
 
-  See `CREATE SERVER` options section for details.  
-  
+  See `CREATE SERVER` options section for details.
+
 - **updatable** as *boolean*, optional, default *true*
 
   This option can allow or disallow write operations on a SQLite table independed of the same server option.
-  
+
 `sqlite_fdw` accepts the following column-level options via the
 `CREATE FOREIGN TABLE` command:
 
@@ -181,7 +181,7 @@ In OS `sqlite_fdw` works as executed code with permissions of user of PostgreSQL
 - **key** as *boolean*, optional, default *false*
 
   Indicates a column as a part of primary key or unique key of SQLite table.
-  
+
 ## IMPORT FOREIGN SCHEMA options
 
 `sqlite_fdw` supports [IMPORT FOREIGN SCHEMA](https://www.postgresql.org/docs/current/sql-importforeignschema.html)
@@ -218,9 +218,9 @@ functions, `sqlite_fdw` provides the following user-callable utility functions:
 - **sqlite_fdw_version()**;
 Returns standard "version integer" as `major version * 10000 + minor version * 100 + bugfix`.
 ```
-sqlite_fdw_version 
+sqlite_fdw_version
 --------------------
-              20300
+              20400
 ```
 Identifier case handling
 ------------------------
@@ -356,7 +356,7 @@ If you want to update tables, please add `OPTIONS (key 'true')` to a primary key
 	  a integer OPTIONS (key 'true'),
 	  b text
 	)
-	SERVER sqlite_server 
+	SERVER sqlite_server
 	OPTIONS (
 	  table 't1_sqlite'
 	);
@@ -438,35 +438,35 @@ Test directory have structure as following:
 
 ```sql
 +---sql
-|   +---11.7
+|   +---12.15
 |   |       filename1.sql
 |   |       filename2.sql
-|   | 
-|   +---12.12
+|   |
+|   +---13.11
 |   |       filename1.sql
 |   |       filename2.sql
-|   | 
-.................  
-|   \---15.0
+|   |
+.................
+|   \---15.3
 |          filename1.sql
 |          filename2.sql
-|          
+|
 \---expected
-|   +---11.7
+|   +---12.15
 |   |       filename1.out
 |   |       filename2.out
-|   | 
-|   +---12.12
+|   |
+|   +---13.11
 |   |       filename1.out
 |   |       filename2.out
-|   | 
-.................  
-|   \---15.0
+|   |
+.................
+|   \---15.3
             filename1.out
             filename2.out
 ```
 The test cases for each version are based on the test of corresponding version of PostgreSQL.
-You can execute test by test.sh directly. 
+You can execute test by test.sh directly.
 The version of PostgreSQL is detected automatically by $(VERSION) variable in Makefile.
 The corresponding sql and expected directory will be used to compare the result. For example, for Postgres 15.0, you can execute "test.sh" directly, and the sql/15.0 and expected/15.0 will be used to compare automatically.
 

connection.c

@@ -710,7 +710,9 @@ sqlite_fdw_get_connections(PG_FUNCTION_ARGS)
 	MemoryContext oldcontext;
 #endif
 
-#if PG_VERSION_NUM >= 150000
+#if PG_VERSION_NUM >= 160000
+	InitMaterializedSRF(fcinfo, 0);
+#elif PG_VERSION_NUM >= 150000
 	SetSingleFuncCall(fcinfo, 0);
 #else
 	/* check to see if caller supports us returning a tuplestore */
@@ -754,17 +756,15 @@ sqlite_fdw_get_connections(PG_FUNCTION_ARGS)
 	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
 	{
 		ForeignServer *server;
-		Datum		values[SQLITE_FDW_GET_CONNECTIONS_COLS];
-		bool		nulls[SQLITE_FDW_GET_CONNECTIONS_COLS];
+		Datum		values[SQLITE_FDW_GET_CONNECTIONS_COLS] = {0};
+		bool		nulls[SQLITE_FDW_GET_CONNECTIONS_COLS] = {0};
 
 		/* We only look for open remote connections */
 		if (!entry->conn)
 			continue;
 
 		server = GetForeignServerExtended(entry->serverid, FSV_MISSING_OK);
 
-		MemSet(values, 0, sizeof(values));
-		MemSet(nulls, 0, sizeof(nulls));
 
 		/*
 		 * The foreign server may have been dropped in current explicit
@@ -811,7 +811,7 @@ sqlite_fdw_get_connections(PG_FUNCTION_ARGS)
 		values[1] = BoolGetDatum(!entry->invalidated);
 #if PG_VERSION_NUM >= 150000
 		tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls);
-#else 
+#else
 		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
 #endif
 	}

deparse.c

@@ -24,6 +24,10 @@
 #include "catalog/pg_operator.h"
 #include "catalog/pg_opfamily.h"
 #include "catalog/pg_proc.h"
+#if PG_VERSION_NUM >= 160000
+#include "catalog/pg_ts_config.h"
+#endif
+#include "catalog/pg_ts_dict.h"
 #include "catalog/pg_type.h"
 #include "commands/defrem.h"
 #include "nodes/nodeFuncs.h"
@@ -108,7 +112,7 @@ static bool sqlite_foreign_expr_walker(Node *node,
 /*
  * Functions to construct string representation of a node tree.
  */
-static void sqlite_deparse_expr(Expr *expr, deparse_expr_cxt *context);
+static void sqlite_deparse_expr(Expr *node, deparse_expr_cxt *context);
 static void sqlite_deparse_var(Var *node, deparse_expr_cxt *context);
 static void sqlite_deparse_const(Const *node, deparse_expr_cxt *context, int showtype);
 static void sqlite_deparse_param(Param *node, deparse_expr_cxt *context);
@@ -356,6 +360,7 @@ sqlite_is_foreign_pathkey(PlannerInfo *root,
 				   PathKey *pathkey)
 {
 	EquivalenceClass *pathkey_ec = pathkey->pk_eclass;
+	EquivalenceMember *em;
 
 	/*
 	 * is_foreign_expr would detect volatile expressions as well, but checking
@@ -368,8 +373,34 @@ sqlite_is_foreign_pathkey(PlannerInfo *root,
 	if (!sqlite_is_builtin(pathkey->pk_opfamily))
 		return false;
 
-	/* can push if a suitable EC member exists */
-	return (sqlite_find_em_for_rel(root, pathkey_ec, baserel) != NULL);
+	/* Find a suitable EC member */
+	em = sqlite_find_em_for_rel(root, pathkey_ec, baserel);
+	if (em)
+	{
+		Oid			oprid;
+		TypeCacheEntry *typentry;
+
+		oprid = get_opfamily_member(pathkey->pk_opfamily,
+									em->em_datatype,
+									em->em_datatype,
+									pathkey->pk_strategy);
+		if (!OidIsValid(oprid))
+			elog(ERROR, "missing operator %d(%u,%u) in opfamily %u",
+				 pathkey->pk_strategy, em->em_datatype, em->em_datatype,
+				 pathkey->pk_opfamily);
+
+		/* See whether operator is default < or > for sort expr's datatype. */
+		typentry = lookup_type_cache(exprType((Node *) em->em_expr),
+									TYPECACHE_LT_OPR | TYPECACHE_GT_OPR);
+
+		/* SQLite does not support USING, so do not push down it */
+		if (oprid != typentry->lt_opr && oprid != typentry->gt_opr)
+			return false;
+	}
+	else
+		return false;
+
+	return true;
 }
 
 /*
@@ -3362,6 +3393,13 @@ sqlite_append_group_by_clause(List *tlist, deparse_expr_cxt *context)
 	 */
 	Assert(!query->groupingSets);
 
+	/*
+	 * We intentionally print query->groupClause not processed_groupClause,
+	 * leaving it to the remote planner to get rid of any redundant GROUP BY
+	 * items again.  This is necessary in case processed_groupClause reduced
+	 * to empty, and in any case the redundancy situation on the remote might
+	 * be different than what we think here.
+	 */
 	foreach(lc, query->groupClause)
 	{
 		SortGroupClause *grp = (SortGroupClause *) lfirst(lc);
@@ -3514,25 +3552,13 @@ static void sqlite_append_order_by_suffix(Oid sortop, Oid sortcoltype,
 	typentry = lookup_type_cache(sortcoltype,
 								 TYPECACHE_LT_OPR | TYPECACHE_GT_OPR);
 
+	/* SQLite does not support USING */
+	Assert (sortop == typentry->lt_opr || sortop == typentry->gt_opr);
+
 	if (sortop == typentry->lt_opr)
 		appendStringInfoString(buf, " ASC");
 	else if (sortop == typentry->gt_opr)
 		appendStringInfoString(buf, " DESC");
-	else
-	{
-		HeapTuple	opertup;
-		Form_pg_operator operform;
-
-		appendStringInfoString(buf, " USING ");
-
-		/* Append operator name. */
-		opertup = SearchSysCache1(OPEROID, ObjectIdGetDatum(sortop));
-		if (!HeapTupleIsValid(opertup))
-			elog(ERROR, "cache lookup failed for operator %u", sortop);
-		operform = (Form_pg_operator) GETSTRUCT(opertup);
-		sqlite_deparse_operator_name(buf, operform);
-		ReleaseSysCache(opertup);
-	}
 
 	if (nulls_first)
 		appendStringInfoString(buf, " NULLS FIRST");
@@ -3736,7 +3762,21 @@ sqlite_get_relation_column_alias_ids(Var *node, RelOptInfo *foreignrel,
 	i = 1;
 	foreach(lc, foreignrel->reltarget->exprs)
 	{
+#if PG_VERSION_NUM >= 160000
+		Var		   *tlvar = (Var *) lfirst(lc);
+
+		/*
+		 * Match reltarget entries only on varno/varattno.  Ideally there
+		 * would be some cross-check on varnullingrels, but it's unclear what
+		 * to do exactly; we don't have enough context to know what that value
+		 * should be.
+		 */
+		if (IsA(tlvar, Var) &&
+			tlvar->varno == node->varno &&
+			tlvar->varattno == node->varattno)
+#else
 		if (equal(lfirst(lc), (Node *) node))
+#endif
 		{
 			*colno = i;
 			return;