Merge branch 'master' into reconcile-rhashimoto-wa-sqlite

Author: ikusteu

.github/ISSUE_TEMPLATE/-do-not-post-anything-other-than-a-bug-report.md

@@ -0,0 +1,10 @@
+---
+name: " Do not post anything other than a bug report"
+about: Issues are only for possible bugs in project code.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ on:
     branches: [ "master" ]
 
 env:
-  EM_VERSION: 3.1.47
+  EM_VERSION: 3.1.61
   EM_CACHE_FOLDER: 'emsdk-cache'
 
 jobs:
@@ -33,7 +33,7 @@ jobs:
     - uses: browser-actions/setup-chrome@v1
       id: setup-chrome
       with:
-        chrome-version: 121
+        chrome-version: 129
     - run: |
           ${{ steps.setup-chrome.outputs.chrome-path }} --version
 
@@ -60,7 +60,7 @@ jobs:
       with:
         path: ${{env.EM_CACHE_FOLDER}}
         key: ${{env.EM_VERSION}}-${{ runner.os }}
-    - uses: mymindstorm/setup-emsdk@v12
+    - uses: mymindstorm/setup-emsdk@v14
       with:
         version: ${{env.EM_VERSION}}
         actions-cache-folder: ${{env.EM_CACHE_FOLDER}}

Makefile

@@ -1,6 +1,6 @@
 # dependencies
-SQLITE_VERSION = version-3.45.0
-SQLITE_TARBALL_URL = https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=${SQLITE_VERSION}
+SQLITE_VERSION = version-3.50.1
+SQLITE_TARBALL_URL = https://www.sqlite.org/src/tarball/$(SQLITE_VERSION)/sqlite.tar.gz
 
 EXTENSION_FUNCTIONS = extension-functions.c
 EXTENSION_FUNCTIONS_URL = https://www.sqlite.org/contrib/download/extension-functions.c?get=25
@@ -18,13 +18,15 @@ CFILES = \
 	main.c \
 	libauthorizer.c \
 	libfunction.c \
+	libhook.c \
 	libprogress.c \
 	libvfs.c \
 	$(CFILES_EXTRA)
 
 JSFILES = \
 	src/libauthorizer.js \
 	src/libfunction.js \
+	src/libhook.js \
 	src/libprogress.js \
 	src/libvfs.js
 
@@ -38,7 +40,7 @@ vpath %.c $(dir.crsql)
 EXPORTED_FUNCTIONS = src/exported_functions.json
 EXPORTED_RUNTIME_METHODS = src/extra_exported_runtime_methods.json
 ASYNCIFY_IMPORTS = src/asyncify_imports.json
-ASYNCIFY_EXPORTS = src/asyncify_exports.json
+JSPI_EXPORTS = src/jspi_exports.json
 
 # intermediate files
 RS_LIB = crsql_bundle
@@ -72,6 +74,7 @@ EMFLAGS_COMMON = \
 	-s INVOKE_RUN \
 	-s ENVIRONMENT="web,worker" \
 	-s STACK_SIZE=512KB \
+	-s WASM_BIGINT=0 \
 	$(EMFLAGS_EXTRA)
 
 EMFLAGS_DEBUG = \
@@ -92,6 +95,7 @@ EMFLAGS_LIBRARIES = \
 	--js-library src/libadapters.js \
 	--post-js src/libauthorizer.js \
 	--post-js src/libfunction.js \
+	--post-js src/libhook.js \
 	--post-js src/libprogress.js \
 	--post-js src/libvfs.js
 
@@ -108,9 +112,9 @@ EMFLAGS_ASYNCIFY_DIST = \
 	-s ASYNCIFY_STACK_SIZE=16384
 
 EMFLAGS_JSPI = \
-	-s ASYNCIFY=2 \
+	-s JSPI \
 	-s ASYNCIFY_IMPORTS=@src/asyncify_imports.json \
-	-s ASYNCIFY_EXPORTS=@src/asyncify_exports.json
+	-s JSPI_EXPORTS=@src/jspi_exports.json
 
 # NOTE: The tests expect the default page size to be 8192 (not 4096 like the sqlite3 default)
 WASQLITE_EXTRA_DEFINES = \

README.md

@@ -1,11 +1,11 @@
-[![wa-sqlite CI](https://github.com/rhashimoto/wa-sqlite/actions/workflows/ci.yml/badge.svg?branch=breaking-changes)](https://github.com/rhashimoto/wa-sqlite/actions/workflows/ci.yml?branch=breaking-changes)
+[![wa-sqlite CI](https://github.com/rhashimoto/wa-sqlite/actions/workflows/ci.yml/badge.svg)](https://github.com/rhashimoto/wa-sqlite/actions/workflows/ci.yml)
 
 # wa-sqlite
 This is a WebAssembly build of SQLite with support for writing SQLite virtual filesystems completely in Javascript. This allows alternative browser storage options such as IndexedDB and Origin Private File System. Applications can opt to use either a synchronous or asynchronous (using Asyncify or JSPI) SQLite library build (an asynchronous build is required for asynchronous extensions).
 
 IndexedDB and several Origin Private File System virtual file systems are among the examples provided as proof of concept. A table comparing the different VFS classes is [here](https://github.com/rhashimoto/wa-sqlite/tree/master/src/examples#vfs-comparison).
 
-[Try the demo](https://rhashimoto.github.io/wa-sqlite/demo/?build=asyncify&config=IDBBatchAtomicVFS&reset) or run [benchmarks](https://rhashimoto.github.io/wa-sqlite/demo/benchmarks/?config=asyncify,IDBBatchAtomicVFS;default,AccessHandlePoolVFS;default,OPFSCoopSyncVFS;asyncify,OPFSAdaptiveVFS;asyncify,OPFSPermutedVFS) with a modern desktop web browser. More information is available in the [FAQ](https://github.com/rhashimoto/wa-sqlite/issues?q=is%3Aissue+label%3Afaq+), [discussion forums](https://github.com/rhashimoto/wa-sqlite/discussions), and [API reference](https://rhashimoto.github.io/wa-sqlite/docs/).
+[Try the demo](https://rhashimoto.github.io/wa-sqlite/demo/?build=asyncify&config=IDBBatchAtomicVFS&reset) or run [benchmarks](https://rhashimoto.github.io/wa-sqlite/demo/benchmarks/?config=asyncify,IDBBatchAtomicVFS;asyncify,IDBMirrorVFS;default,AccessHandlePoolVFS;default,OPFSCoopSyncVFS;asyncify,OPFSAdaptiveVFS;asyncify,OPFSPermutedVFS) with a modern desktop web browser. More information is available in the [FAQ](https://github.com/rhashimoto/wa-sqlite/issues?q=is%3Aissue+label%3Afaq+), [discussion forums](https://github.com/rhashimoto/wa-sqlite/discussions), and [API reference](https://rhashimoto.github.io/wa-sqlite/docs/).
 
 ## Build
 The primary motivation for this project is to enable additions to SQLite with only Javascript. Most developers should be able to use the pre-built artifacts in
@@ -19,7 +19,7 @@ If you do want to build yourself, here are the prerequisites:
 
 * Building on Debian Linux is known to work, compatibility with other platforms is unknown.
 * `yarn` - If you use a different package manager (e.g. `npm`) then file paths in the demo will need adjustment.
-* [Emscripten SDK](https://emscripten.org/docs/getting_started/downloads.html) 3.1.47+.
+* [Emscripten SDK](https://emscripten.org/docs/getting_started/downloads.html) 3.1.61+.
 * `curl`, `make`, `openssl`, `sed`, `tclsh`, `unzip`
 
 Here are the build steps:
@@ -64,10 +64,10 @@ To serve the demo directly from the source tree:
 
 The demo page provides access to databases on multiple VFS implementations. Query parameters on the demo page URL can be used to specify the configuration and initial state:
 
-| *Parameter* | *Purpose* | *Values* | *Default* |
+| Parameter | Purpose | Values | Default |
 |----|----|----|----|
 | build | Emscripten build type | default, asyncify, jspi | default |
-| config | select VFS | MemoryVFS, MemoryAsyncVFS, IDBBatchAtomicVFS, AccessHandlePoolVFS, OPFSAdaptiveVFS, OPFSCoopSyncVFS, OPFSPermutedVFS | uses SQLite internal memory |
+| config | select VFS | MemoryVFS, MemoryAsyncVFS, IDBBatchAtomicVFS, IDBMirrorVFS, AccessHandlePoolVFS, OPFSAdaptiveVFS, OPFSAnyContextVFS, OPFSCoopSyncVFS, OPFSPermutedVFS | uses SQLite internal memory |
 | reset | clear persistent storage | | |
 
 For convenience, if any text region is selected in the editor, only that region will be executed. In addition, the editor contents are restored across page reloads using browser localStorage.

demo/contention/contention-worker.js

@@ -37,6 +37,10 @@ const BUILDS = new Map([
     vfsModule: '../../src/examples/IDBBatchAtomicVFS.js',
     vfsOptions: { lockPolicy: 'shared+hint' }
   },
+  {
+    name: 'IDBMirrorVFS',
+    vfsModule: '../../src/examples/IDBMirrorVFS.js',
+  },
   {
     name: 'OPFSAdaptiveVFS',
     vfsModule: '../../src/examples/OPFSAdaptiveVFS.js',

demo/demo-worker.js

@@ -13,6 +13,7 @@ const BUILDS = new Map([
  * @property {string} name
  * @property {string} vfsModule path of the VFS module
  * @property {string} [vfsClassName] name of the VFS class
+ * @property {string} [vfsName] name of the VFS instance
  * @property {object} [vfsOptions] VFS constructor arguments
  */
 
@@ -34,6 +35,11 @@ const BUILDS = new Map([
     vfsModule: '../src/examples/IDBBatchAtomicVFS.js',
     vfsOptions: { lockPolicy: 'shared+hint' }
   },
+  {
+    name: 'IDBMirrorVFS',
+    vfsModule: '../src/examples/IDBMirrorVFS.js',
+    vfsName: 'demo-mirror'
+  },
   {
     name: 'OPFSAdaptiveVFS',
     vfsModule: '../src/examples/OPFSAdaptiveVFS.js',
@@ -70,7 +76,7 @@ maybeReset().then(async () => {
   const config = VFS_CONFIGS.get(configName);
 
   const dbName = searchParams.get('dbName') ?? 'hello';
-  const vfsName = searchParams.get('vfsName') ?? 'demo';
+  const vfsName = searchParams.get('vfsName') ?? config.vfsName ?? 'demo';
 
   // Instantiate SQLite.
   const { default: moduleFactory } = await import(BUILDS.get(buildName));

demo/demo.js

@@ -77,6 +77,7 @@ window.addEventListener('DOMContentLoaded', async function() {
     let time = performance.now();
     worker.postMessage(queries);
     worker.addEventListener('message', async function(event) {
+      timestamp.textContent += ` ${(performance.now() - time).toFixed(1)} milliseconds`;
       if (event.data.results) {
         // Format the results as tables.
         event.data.results
@@ -85,7 +86,6 @@ window.addEventListener('DOMContentLoaded', async function() {
       } else {
         output.innerHTML = `<pre>${event.data.error.message}</pre>`;
       }
-      timestamp.textContent += ` ${Math.trunc(performance.now() - time) / 1000} seconds`;
       button.disabled = false;
     }, { once: true });
   });

demo/file/index.js

@@ -1,5 +1,6 @@
 import * as VFS from "../../src/VFS.js";
-import { IDBBatchAtomicVFS } from "../../src/examples/IDBBatchAtomicVFS.js";
+import { IDBBatchAtomicVFS as MyVFS } from "../../src/examples/IDBBatchAtomicVFS.js";
+// import { IDBMirrorVFS as MyVFS } from "../../src/examples/IDBMirrorVFS.js";
 
 const SEARCH_PARAMS = new URLSearchParams(location.search);
 const IDB_NAME = SEARCH_PARAMS.get('idb') ?? 'sqlite-vfs';
@@ -43,7 +44,7 @@ document.getElementById('file-fetch').addEventListener('click', async () => {
   let vfs;
   try {
     log(`Importing to IndexedDB ${IDB_NAME}, path ${DB_NAME}`);
-    vfs = await IDBBatchAtomicVFS.create(IDB_NAME, null);
+    vfs = await MyVFS.create(IDB_NAME, null);
 
     // @ts-ignore
     const importURL = document.getElementById('file-url').value;
@@ -69,7 +70,7 @@ document.getElementById('file-import').addEventListener('change', async event =>
   let vfs;
   try {
     log(`Importing to IndexedDB ${IDB_NAME}, path ${DB_NAME}`);
-    vfs = await IDBBatchAtomicVFS.create(IDB_NAME, null);
+    vfs = await MyVFS.create(IDB_NAME, null);
     // @ts-ignore
     await importDatabase(vfs, DB_NAME, event.target.files[0].stream());
     log('Import complete');
@@ -87,7 +88,7 @@ document.getElementById('file-import').addEventListener('change', async event =>
 });
 
 /**
- * @param {IDBBatchAtomicVFS} vfs 
+ * @param {MyVFS} vfs 
  * @param {string} path 
  * @param {ReadableStream} stream 
  */

demo/file/service-worker.js

@@ -1,5 +1,6 @@
 import * as VFS from "../../src/VFS.js";
-import { IDBBatchAtomicVFS } from "../../src/examples/IDBBatchAtomicVFS.js";
+import { IDBBatchAtomicVFS as MyVFS } from "../../src/examples/IDBBatchAtomicVFS.js";
+// import { IDBMirrorVFS as MyVFS } from "../../src/examples/IDBMirrorVFS.js";
 
 // Install the service worker as soon as possible.
 globalThis.addEventListener('install', (/** @type {ExtendableEvent} */ event) => {
@@ -26,7 +27,7 @@ globalThis.addEventListener('fetch', async (/** @type {FetchEvent} */ event) =>
 
   return event.respondWith((async () => {
     // Create the VFS and streaming source using the request parameters.
-    const vfs = await IDBBatchAtomicVFS.create(url.searchParams.get('idb'), null);
+    const vfs = await MyVFS.create(url.searchParams.get('idb'), null);
     const path = url.searchParams.get('db');
     const source = new DatabaseSource(vfs, path);
 

demo/file/verifier.js

@@ -1,6 +1,7 @@
 import SQLiteESMFactory from '../../dist/wa-sqlite-async.mjs';
 import * as SQLite from '../../src/sqlite-api.js';
-import { IDBBatchAtomicVFS } from '../../src/examples/IDBBatchAtomicVFS.js';
+import { IDBBatchAtomicVFS as MyVFS } from "../../src/examples/IDBBatchAtomicVFS.js";
+// import { IDBMirrorVFS as MyVFS } from "../../src/examples/IDBMirrorVFS.js";
 
 const SEARCH_PARAMS = new URLSearchParams(location.search);
 const IDB_NAME = SEARCH_PARAMS.get('idb') ?? 'sqlite-vfs';
@@ -10,7 +11,7 @@ const DB_NAME = SEARCH_PARAMS.get('db') ?? 'sqlite.db';
   const module = await SQLiteESMFactory();
   const sqlite3 = SQLite.Factory(module);
 
-  const vfs = await IDBBatchAtomicVFS.create(IDB_NAME, module);
+  const vfs = await MyVFS.create(IDB_NAME, module);
   // @ts-ignore
   sqlite3.vfs_register(vfs, true);