Tdrz/backend pgdump (#52)

Use emscripten built pg_dump.

---------

Co-authored-by: tudor <tudor@swisstch.com>

Author: tdrz

.gitignore

@@ -46,7 +46,10 @@ lib*.pc
 /dist
 
 # emscripten build excludes
+*.js
+*.data
 *.cjs
+pg_config.mjs
 *.wasm
 pglite.data
 pglite.js

build-pglite.sh

@@ -37,9 +37,25 @@ else
     echo "$CONFIG_STATUS exists and is newer than $REF_FILE. ./configure will NOT be run."
 fi
 
+# we define here "all" emscripten flags in order to allow native builds (like libpglite)
+EXPORTED_RUNTIME_METHODS="addFunction,removeFunction,FS,MEMFS"
+PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \
+-sSUPPORT_LONGJMP=emscripten \
+-sFORCE_FILESYSTEM=1 \
+-sNO_EXIT_RUNTIME=0 -sENVIRONMENT=node,web,worker \
+-sMAIN_MODULE=2 -sMODULARIZE=1 -sEXPORT_ES6=1 \
+-sEXPORT_NAME=Module -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH \
+-sERROR_ON_UNDEFINED_SYMBOLS=0 \
+-sEXPORTED_RUNTIME_METHODS=$EXPORTED_RUNTIME_METHODS \
+-sTOTAL_MEMORY=32MB \
+--embed-file $(pwd)/other/PGPASSFILE@/home/web_user/.pgpass"
+
 # Step 1: configure the project
 if [ "$RUN_CONFIGURE" = true ]; then
-    LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.cjs --disable-spinlocks --disable-largefile --without-llvm  --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2 --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; }
+    LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" \
+    LDFLAGS_SL="-sSIDE_MODULE=1" \
+    LDFLAGS_EX=$PGLITE_EMSCRIPTEN_FLAGS \
+    CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.js --host aarch64-unknown-linux-gnu --disable-spinlocks --disable-largefile --without-llvm  --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2:$(pwd)/pglite/includes --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; }
 else
     echo "Warning: configure has not been run because RUN_CONFIGURE=${RUN_CONFIGURE}"
 fi
@@ -49,7 +65,7 @@ emmake make PORTNAME=emscripten -j || { echo 'error: emmake make PORTNAME=emscri
 emmake make PORTNAME=emscripten install || { echo 'error: emmake make PORTNAME=emscripten install' ; exit 22; }
 
 # Step 3.1: make all contrib extensions - do not install
-emmake make PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C contrib/ -j || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ -j' ; exit 31; }
+emmake make PORTNAME=emscripten -C contrib/ -j || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ -j' ; exit 31; }
 # Step 3.2: make dist contrib extensions - this will create an archive for each extension
 emmake make PORTNAME=emscripten -C contrib/ dist || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ dist' ; exit 32; }
 # the above will also create a file with the imports that each extension needs - we pass these as input in the next step for emscripten to keep alive
@@ -58,20 +74,18 @@ emmake make PORTNAME=emscripten -C contrib/ dist || { echo 'error: emmake make P
 SAVE_PATH=$PATH
 PATH=$PATH:$INSTALL_FOLDER/bin
 emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite || { echo 'error: emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 41; }
-emmake make OPTFLAGS="" PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C pglite/ dist || { echo 'error: make OPTFLAGS="" PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C pglite/ dist ' ; exit 42; }
+emmake make OPTFLAGS="" PORTNAME=emscripten -C pglite/ dist || { echo 'error: make OPTFLAGS="" PORTNAME=emscripten -C pglite/ dist ' ; exit 42; }
 PATH=$SAVE_PATH
 
 # Step 5: make and install pglite
-# we define here "all" emscripten flags in order to allow native builds (like libpglite)
 EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,addFunction,removeFunction"
 PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \
 -sSUPPORT_LONGJMP=emscripten \
 -sFORCE_FILESYSTEM=1 \
 -sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web,worker \
 -sMAIN_MODULE=2 -sMODULARIZE=1 -sEXPORT_ES6=1 \
 -sEXPORT_NAME=Module -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH \
--sERROR_ON_UNDEFINED_SYMBOLS=1 \
+-sERROR_ON_UNDEFINED_SYMBOLS=0 \
 -sEXPORTED_RUNTIME_METHODS=$EXPORTED_RUNTIME_METHODS"
-
 # Building pglite itself needs to be the last step because of the PRELOAD_FILES parameter (a list of files and folders) need to be available.
 PGLITE_CFLAGS="$PGLITE_CFLAGS $PGLITE_EMSCRIPTEN_FLAGS" emmake make PORTNAME=emscripten -j -C src/backend/ install-pglite || { echo 'emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 51; }

pglite-wasm/interactive_one.c

@@ -1,35 +1,10 @@
 #include <unistd.h>  // access, unlink
 #define PGL_LOOP
-#if defined(__wasi__)
-// volatile sigjmp_buf void*;
-#else
 volatile sigjmp_buf local_sigjmp_buf;
-#endif
 
 // track back how many ex raised in steps of the loop until sucessfull clear_error
 volatile int canary_ex = 0;
 
-// read FROM JS
-// (i guess return number of bytes written)
-// ssize_t pglite_read(/* ignored */ int socket, void *buffer, size_t length,/* ignored */ int flags,/* ignored */ void *address,/* ignored */ socklen_t *address_len);
-//typedef ssize_t (*pglite_read_t)(/* ignored */ int socket, void *buffer, size_t length,/* ignored */ int flags,/* ignored */ void *address,/* ignored */ unsigned int *address_len);
-typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length);
-extern pglite_read_t pglite_read;
-
-// write TO JS
-// (i guess return number of bytes read)
-// ssize_t pglite_write(/* ignored */ int sockfd, const void *buf, size_t len, /* ignored */ int flags);
-// typedef ssize_t (*pglite_write_t)(/* ignored */ int sockfd, const void *buf, size_t len, /* ignored */ int flags);
-typedef ssize_t (*pglite_write_t)(void *buffer, size_t length);
-extern pglite_write_t pglite_write;
-
-__attribute__((export_name("set_read_write_cbs")))
-void
-set_read_write_cbs(pglite_read_t read_cb, pglite_write_t write_cb) {
-    pglite_read = read_cb;
-    pglite_write = write_cb;
-}
-
 extern void AbortTransaction(void);
 extern void CleanupTransaction(void);
 extern void ClientAuthentication(Port *port);

pglite-wasm/pgl_mains.c

@@ -218,13 +218,9 @@ PDEBUG("# 164:" __FILE__);
     initStringInfo(&row_description_buf);
     MemoryContextSwitchTo(TopMemoryContext);
 
-#if defined(__wasi__)
-    puts("# 210: sjlj exception handler off in initdb-wasi");
-#else
 #   define INITDB_SINGLE
 #   include "pgl_sjlj.c"
 #   undef INITDB_SINGLE
-#endif // sjlj
 
     if (!ignore_till_sync)
         send_ready_for_query = true;	/* initially, or after error */

pglite-wasm/pgl_sjlj.c

@@ -1,6 +1,3 @@
-#if defined(__wasi__)
-    PDEBUG("# 2:" __FILE__ ": sjlj exception handler off");
-#else
 	if (sigsetjmp(local_sigjmp_buf, 1) != 0)
 	{
 #if !defined(INITDB_SINGLE)
@@ -67,5 +64,4 @@
     }
 
 	PG_exception_stack = &local_sigjmp_buf;
-#endif
 

pglite-wasm/pgl_stubs.h

@@ -1,15 +1,6 @@
 #pragma once
 
-
-// wasi only stubs
-#if defined(__wasi__)
-#   undef PQEXPBUFFER_H
-#   include "../src/interfaces/libpq/pqexpbuffer.h"
-
-#else
-#   include "../src/interfaces/libpq/pqexpbuffer.h"
-#endif
-
+#include "../src/interfaces/libpq/pqexpbuffer.h"
 
 // option_parse_int parse_sync_method
 #include "../src/fe_utils/option_utils.c"

pglite/includes/pglite-comm.h

@@ -0,0 +1,88 @@
+#if defined(__EMSCRIPTEN__)
+
+#ifndef PGLITE_COMM_H
+#define PGLITE_COMM_H
+
+#include <emscripten/emscripten.h>
+
+volatile int querylen = 0;
+volatile FILE* queryfp = NULL;
+
+// read FROM JS
+// (i guess return number of bytes written)
+// ssize_t pglite_read(/* ignored */ int socket, void *buffer, size_t length,/* ignored */ int flags,/* ignored */ void *address,/* ignored */ socklen_t *address_len);
+//typedef ssize_t (*pglite_read_t)(/* ignored */ int socket, void *buffer, size_t length,/* ignored */ int flags,/* ignored */ void *address,/* ignored */ unsigned int *address_len);
+typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length);
+pglite_read_t pglite_read;
+
+// write TO JS
+// (i guess return number of bytes read)
+// ssize_t pglite_write(/* ignored */ int sockfd, const void *buf, size_t len, /* ignored */ int flags);
+// typedef ssize_t (*pglite_write_t)(/* ignored */ int sockfd, const void *buf, size_t len, /* ignored */ int flags);
+typedef ssize_t (*pglite_write_t)(void *buffer, size_t length);
+pglite_write_t pglite_write;
+
+__attribute__((export_name("set_read_write_cbs")))
+void
+set_read_write_cbs(pglite_read_t read_cb, pglite_write_t write_cb) {
+    pglite_read = read_cb;
+    pglite_write = write_cb;
+}
+
+int EMSCRIPTEN_KEEPALIVE fcntl(int __fd, int __cmd, ...) {
+	// dummy 
+	return 0;
+}
+
+int EMSCRIPTEN_KEEPALIVE setsockopt(int __fd, int __level, int __optname,
+	const void *__optval, socklen_t __optlen) {
+		// dummy 
+		return 0;
+}
+
+int EMSCRIPTEN_KEEPALIVE getsockopt(int __fd, int __level, int __optname,
+	void *__restrict __optval,
+	socklen_t *__restrict __optlen) {
+		// dummy 
+		return 0;
+}
+
+int EMSCRIPTEN_KEEPALIVE getsockname(int __fd, struct sockaddr * __addr,
+	socklen_t *__restrict __len) {
+		// dummy 
+		return 0;
+	}
+
+ssize_t EMSCRIPTEN_KEEPALIVE
+	recv(int __fd, void *__buf, size_t __n, int __flags) {
+		ssize_t got = pglite_read(__buf, __n);
+		return got;
+	}
+
+ssize_t EMSCRIPTEN_KEEPALIVE
+	send(int __fd, const void *__buf, size_t __n, int __flags) {
+		ssize_t wrote = pglite_write(__buf, __n);
+		return wrote;
+	}
+
+int EMSCRIPTEN_KEEPALIVE
+    connect(int socket, const struct sockaddr *address, socklen_t address_len) {
+		// dummy
+		return 0;
+	}
+
+struct pollfd {
+    int   fd;         /* file descriptor */
+    short events;     /* requested events */
+	short revents;    /* returned events */
+};
+
+int EMSCRIPTEN_KEEPALIVE
+    poll(struct pollfd fds[], ssize_t nfds, int timeout) {
+		// dummy
+		return nfds;
+	}
+
+#endif
+
+#endif // ifndef PGLITE_COMM_H

src/backend/libpq/auth.c

@@ -1856,7 +1856,7 @@ auth_peer(hbaPort *port)
 {
 	uid_t		uid;
 	gid_t		gid;
-#if !defined(WIN32) && !defined(__wasi__)
+#if !defined(WIN32)
 	struct passwd *pw;
 	int			ret;
 #endif
@@ -1875,7 +1875,7 @@ auth_peer(hbaPort *port)
 		return STATUS_ERROR;
 	}
 
-#if !defined(WIN32) && !defined(__wasi__)
+#if !defined(WIN32)
 	errno = 0;					/* clear errno before call */
 	pw = getpwuid(uid);
 	if (!pw)

src/backend/libpq/be-fsstubs.c

@@ -150,12 +150,8 @@ be_lo_close(PG_FUNCTION_ARGS)
  *
  *****************************************************************************/
 
-#if defined(__EMSCRIPTEN__) || defined(__wasi__)
-static int
-#else
 int
-#endif
-lo_read3(int fd, char *buf, int len)
+lo_read(int fd, char *buf, int len)
 {
 	int			status;
 	LargeObjectDesc *lobj;
@@ -182,12 +178,8 @@ lo_read3(int fd, char *buf, int len)
 	return status;
 }
 
-#if defined(__EMSCRIPTEN__) || defined(__wasi__)
-static int
-#else
 int
-#endif
-lo_write3(int fd, const char *buf, int len)
+lo_write(int fd, const char *buf, int len)
 {
 	int			status;
 	LargeObjectDesc *lobj;
@@ -198,7 +190,7 @@ lo_write3(int fd, const char *buf, int len)
 				 errmsg("invalid large-object descriptor: %d", fd)));
 	lobj = cookies[fd];
 
-	/* see comment in lo_read3() */
+	/* see comment in lo_read() */
 	if ((lobj->flags & IFS_WRLOCK) == 0)
 		ereport(ERROR,
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
@@ -373,7 +365,7 @@ be_loread(PG_FUNCTION_ARGS)
 		len = 0;
 
 	retval = (bytea *) palloc(VARHDRSZ + len);
-	totalread = lo_read3(fd, VARDATA(retval), len);
+	totalread = lo_read(fd, VARDATA(retval), len);
 	SET_VARSIZE(retval, totalread + VARHDRSZ);
 
 	PG_RETURN_BYTEA_P(retval);
@@ -390,7 +382,7 @@ be_lowrite(PG_FUNCTION_ARGS)
 	PreventCommandIfReadOnly("lowrite()");
 
 	bytestowrite = VARSIZE_ANY_EXHDR(wbuf);
-	totalwritten = lo_write3(fd, VARDATA_ANY(wbuf), bytestowrite);
+	totalwritten = lo_write(fd, VARDATA_ANY(wbuf), bytestowrite);
 	PG_RETURN_INT32(totalwritten);
 }
 
@@ -568,7 +560,7 @@ lo_truncate_internal(int32 fd, int64 len)
 				 errmsg("invalid large-object descriptor: %d", fd)));
 	lobj = cookies[fd];
 
-	/* see comment in lo_read3() */
+	/* see comment in lo_read() */
 	if ((lobj->flags & IFS_WRLOCK) == 0)
 		ereport(ERROR,
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),

src/backend/libpq/be-secure.c

@@ -32,6 +32,10 @@
 #include "tcop/tcopprot.h"
 #include "utils/wait_event.h"
 
+#if defined(__EMSCRIPTEN__)
+#include "pglite-comm.h"
+#endif
+
 char	   *ssl_library;
 char	   *ssl_cert_file;
 char	   *ssl_key_file;