Merge pull request #300 from powersync-ja/raw-tables

Raw tables

Author: simolus3

packages/powersync_core/lib/src/database/native/native_powersync_database.dart

@@ -1,4 +1,5 @@
 import 'dart:async';
+import 'dart:convert';
 import 'dart:isolate';
 import 'package:meta/meta.dart';
 
@@ -83,8 +84,12 @@ class PowerSyncDatabaseImpl
     DefaultSqliteOpenFactory factory =
         // ignore: deprecated_member_use_from_same_package
         PowerSyncOpenFactory(path: path, sqliteSetup: sqliteSetup);
-    return PowerSyncDatabaseImpl.withFactory(factory,
-        schema: schema, maxReaders: maxReaders, logger: logger);
+    return PowerSyncDatabaseImpl.withFactory(
+      factory,
+      schema: schema,
+      maxReaders: maxReaders,
+      logger: logger,
+    );
   }
 
   /// Open a [PowerSyncDatabase] with a [PowerSyncOpenFactory].
@@ -96,22 +101,29 @@ class PowerSyncDatabaseImpl
   ///
   /// [logger] defaults to [autoLogger], which logs to the console in debug builds.
   factory PowerSyncDatabaseImpl.withFactory(
-      DefaultSqliteOpenFactory openFactory,
-      {required Schema schema,
-      int maxReaders = SqliteDatabase.defaultMaxReaders,
-      Logger? logger}) {
+    DefaultSqliteOpenFactory openFactory, {
+    required Schema schema,
+    int maxReaders = SqliteDatabase.defaultMaxReaders,
+    Logger? logger,
+  }) {
     final db = SqliteDatabase.withFactory(openFactory, maxReaders: maxReaders);
     return PowerSyncDatabaseImpl.withDatabase(
-        schema: schema, database: db, logger: logger);
+      schema: schema,
+      database: db,
+      logger: logger,
+    );
   }
 
   /// Open a PowerSyncDatabase on an existing [SqliteDatabase].
   ///
   /// Migrations are run on the database when this constructor is called.
   ///
   /// [logger] defaults to [autoLogger], which logs to the console in debug builds.s
-  PowerSyncDatabaseImpl.withDatabase(
-      {required this.schema, required this.database, Logger? logger}) {
+  PowerSyncDatabaseImpl.withDatabase({
+    required this.schema,
+    required this.database,
+    Logger? logger,
+  }) {
     this.logger = logger ?? autoLogger;
     isInitialized = baseInit();
   }
@@ -247,6 +259,7 @@ class PowerSyncDatabaseImpl
         options,
         crudMutex.shared,
         syncMutex.shared,
+        jsonEncode(schema),
       ),
       debugName: 'Sync ${database.openFactory.path}',
       onError: receiveUnhandledErrors.sendPort,
@@ -290,13 +303,15 @@ class _PowerSyncDatabaseIsolateArgs {
   final ResolvedSyncOptions options;
   final SerializedMutex crudMutex;
   final SerializedMutex syncMutex;
+  final String schemaJson;
 
   _PowerSyncDatabaseIsolateArgs(
     this.sPort,
     this.dbRef,
     this.options,
     this.crudMutex,
     this.syncMutex,
+    this.schemaJson,
   );
 }
 
@@ -392,6 +407,7 @@ Future<void> _syncIsolate(_PowerSyncDatabaseIsolateArgs args) async {
     final storage = BucketStorage(connection);
     final sync = StreamingSyncImplementation(
       adapter: storage,
+      schemaJson: args.schemaJson,
       connector: InternalConnector(
         getCredentialsCached: getCredentialsCached,
         prefetchCredentials: prefetchCredentials,

packages/powersync_core/lib/src/database/powersync_database.dart

@@ -32,19 +32,21 @@ abstract class PowerSyncDatabase
   /// A maximum of [maxReaders] concurrent read transactions are allowed.
   ///
   /// [logger] defaults to [autoLogger], which logs to the console in debug builds.
-  factory PowerSyncDatabase(
-      {required Schema schema,
-      required String path,
-      Logger? logger,
-      @Deprecated("Use [PowerSyncDatabase.withFactory] instead.")
-      // ignore: deprecated_member_use_from_same_package
-      SqliteConnectionSetup? sqliteSetup}) {
+  factory PowerSyncDatabase({
+    required Schema schema,
+    required String path,
+    Logger? logger,
+    @Deprecated("Use [PowerSyncDatabase.withFactory] instead.")
+    // ignore: deprecated_member_use_from_same_package
+    SqliteConnectionSetup? sqliteSetup,
+  }) {
     return PowerSyncDatabaseImpl(
-        schema: schema,
-        path: path,
-        logger: logger,
-        // ignore: deprecated_member_use_from_same_package
-        sqliteSetup: sqliteSetup);
+      schema: schema,
+      path: path,
+      logger: logger,
+      // ignore: deprecated_member_use_from_same_package
+      sqliteSetup: sqliteSetup,
+    );
   }
 
   /// Open a [PowerSyncDatabase] with a [PowerSyncOpenFactory].
@@ -55,12 +57,18 @@ abstract class PowerSyncDatabase
   /// Subclass [PowerSyncOpenFactory] to add custom logic to this process.
   ///
   /// [logger] defaults to [autoLogger], which logs to the console in debug builds.
-  factory PowerSyncDatabase.withFactory(DefaultSqliteOpenFactory openFactory,
-      {required Schema schema,
-      int maxReaders = SqliteDatabase.defaultMaxReaders,
-      Logger? logger}) {
-    return PowerSyncDatabaseImpl.withFactory(openFactory,
-        schema: schema, maxReaders: maxReaders, logger: logger);
+  factory PowerSyncDatabase.withFactory(
+    DefaultSqliteOpenFactory openFactory, {
+    required Schema schema,
+    int maxReaders = SqliteDatabase.defaultMaxReaders,
+    Logger? logger,
+  }) {
+    return PowerSyncDatabaseImpl.withFactory(
+      openFactory,
+      schema: schema,
+      maxReaders: maxReaders,
+      logger: logger,
+    );
   }
 
   /// Open a PowerSyncDatabase on an existing [SqliteDatabase].

packages/powersync_core/lib/src/database/powersync_database_impl_stub.dart

@@ -82,10 +82,11 @@ class PowerSyncDatabaseImpl
   /// Migrations are run on the database when this constructor is called.
   ///
   /// [logger] defaults to [autoLogger], which logs to the console in debug builds.s
-  factory PowerSyncDatabaseImpl.withDatabase(
-      {required Schema schema,
-      required SqliteDatabase database,
-      Logger? logger}) {
+  factory PowerSyncDatabaseImpl.withDatabase({
+    required Schema schema,
+    required SqliteDatabase database,
+    Logger? logger,
+  }) {
     throw UnimplementedError();
   }
 

packages/powersync_core/lib/src/database/powersync_db_mixin.dart

@@ -297,6 +297,13 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection {
       params: params,
     );
 
+    if (schema.rawTables.isNotEmpty &&
+        resolvedOptions.source.syncImplementation !=
+            SyncClientImplementation.rust) {
+      throw UnsupportedError(
+          'Raw tables are only supported by the Rust client.');
+    }
+
     // ignore: deprecated_member_use_from_same_package
     clientParams = params;
     var thisConnectAborter = AbortController();

packages/powersync_core/lib/src/database/web/web_powersync_database.dart

@@ -1,4 +1,5 @@
 import 'dart:async';
+import 'dart:convert';
 import 'package:meta/meta.dart';
 import 'package:http/browser_client.dart';
 import 'package:logging/logging.dart';
@@ -75,8 +76,12 @@ class PowerSyncDatabaseImpl
       SqliteConnectionSetup? sqliteSetup}) {
     // ignore: deprecated_member_use_from_same_package
     DefaultSqliteOpenFactory factory = PowerSyncOpenFactory(path: path);
-    return PowerSyncDatabaseImpl.withFactory(factory,
-        maxReaders: maxReaders, logger: logger, schema: schema);
+    return PowerSyncDatabaseImpl.withFactory(
+      factory,
+      maxReaders: maxReaders,
+      logger: logger,
+      schema: schema,
+    );
   }
 
   /// Open a [PowerSyncDatabase] with a [PowerSyncOpenFactory].
@@ -94,16 +99,22 @@ class PowerSyncDatabaseImpl
       Logger? logger}) {
     final db = SqliteDatabase.withFactory(openFactory, maxReaders: 1);
     return PowerSyncDatabaseImpl.withDatabase(
-        schema: schema, logger: logger, database: db);
+      schema: schema,
+      logger: logger,
+      database: db,
+    );
   }
 
   /// Open a PowerSyncDatabase on an existing [SqliteDatabase].
   ///
   /// Migrations are run on the database when this constructor is called.
   ///
   /// [logger] defaults to [autoLogger], which logs to the console in debug builds.
-  PowerSyncDatabaseImpl.withDatabase(
-      {required this.schema, required this.database, Logger? logger}) {
+  PowerSyncDatabaseImpl.withDatabase({
+    required this.schema,
+    required this.database,
+    Logger? logger,
+  }) {
     if (logger != null) {
       this.logger = logger;
     } else {
@@ -141,6 +152,7 @@ class PowerSyncDatabaseImpl
 
       sync = StreamingSyncImplementation(
         adapter: storage,
+        schemaJson: jsonEncode(schema),
         connector: InternalConnector.wrap(connector, this),
         crudUpdateTriggerStream: crudStream,
         options: options,

packages/powersync_core/lib/src/schema.dart

@@ -7,11 +7,23 @@ import 'schema_logic.dart';
 /// No migrations are required on the client.
 class Schema {
   /// List of tables in the schema.
+  ///
+  /// When opening a PowerSync database, these tables will be created and
+  /// migrated automatically.
   final List<Table> tables;
 
-  const Schema(this.tables);
+  /// A list of [RawTable]s in addition to PowerSync-managed [tables].
+  ///
+  /// Raw tables give users full control over the SQLite tables, but that
+  /// includes the responsibility to create those tables and to write migrations
+  /// for them.
+  ///
+  /// For more information on raw tables, see [RawTable] and [the documentation](https://docs.powersync.com/usage/use-case-examples/raw-tables).
+  final List<RawTable> rawTables;
+
+  const Schema(this.tables, {this.rawTables = const []});
 
-  Map<String, dynamic> toJson() => {'tables': tables};
+  Map<String, dynamic> toJson() => {'raw_tables': rawTables, 'tables': tables};
 
   void validate() {
     Set<String> tableNames = {};
@@ -315,6 +327,120 @@ class Column {
   Map<String, dynamic> toJson() => {'name': name, 'type': type.sqlite};
 }
 
+/// A raw table, defined by the user instead of being managed by PowerSync.
+///
+/// Any ordinary SQLite table can be defined as a raw table, which enables:
+///
+/// - More performant queries, since data is stored in typed rows instead of the
+///   schemaless JSON view PowerSync uses by default.
+/// - More control over the table, since custom column constraints can be used
+///   in its definition.
+///
+/// PowerSync doesn't know anything about the internal structure of raw tables -
+/// instead, it relies on user-defined [put] and [delete] statements to sync
+/// data into them.
+///
+/// When using raw tables, you are responsible for creating and migrating them
+/// when they've changed. Further, triggers are necessary to collect local
+/// writes to those tables. For more information, see
+/// [the documentation](https://docs.powersync.com/usage/use-case-examples/raw-tables).
+///
+/// Note that raw tables are only supported by the Rust sync client, which needs
+/// to be enabled when connecting with raw tables.
+final class RawTable {
+  /// The name of the table as used by the sync service.
+  ///
+  /// This doesn't necessarily have to match the name of the SQLite table that
+  /// [put] and [delete] write to. Instead, it's used by the sync client to
+  /// identify which statements to use when it encounters sync operations for
+  /// this table.
+  final String name;
+
+  /// A statement responsible for inserting or updating a row in this raw table
+  /// based on data from the sync service.
+  ///
+  /// See [PendingStatement] for details.
+  final PendingStatement put;
+
+  /// A statement responsible for deleting a row based on its PowerSync id.
+  ///
+  /// See [PendingStatement] for details. Note that [PendingStatementValue]s
+  /// used here must all be [PendingStatementValue.id].
+  final PendingStatement delete;
+
+  const RawTable({
+    required this.name,
+    required this.put,
+    required this.delete,
+  });
+
+  Map<String, dynamic> toJson() => {
+        'name': name,
+        'put': put,
+        'delete': delete,
+      };
+}
+
+/// An SQL statement to be run by the sync client against raw tables.
+///
+/// Since raw tables are managed by the user, PowerSync can't know how to apply
+/// serverside changes to them. These statements bridge raw tables and PowerSync
+/// by providing upserts and delete statements.
+///
+/// For more information, see [the documentation](https://docs.powersync.com/usage/use-case-examples/raw-tables)
+final class PendingStatement {
+  /// The SQL statement to run to upsert or delete data from a raw table.
+  final String sql;
+
+  /// A list of value identifiers for parameters in [sql].
+  ///
+  /// Put statements can use both [PendingStatementValue.id] and
+  /// [PendingStatementValue.column], whereas delete statements can only use
+  /// [PendingStatementValue.id].
+  final List<PendingStatementValue> params;
+
+  PendingStatement({required this.sql, required this.params});
+
+  Map<String, dynamic> toJson() => {
+        'sql': sql,
+        'params': params,
+      };
+}
+
+/// A description of a value that will be resolved in the sync client when
+/// running a [PendingStatement] for a [RawTable].
+sealed class PendingStatementValue {
+  /// A value that is bound to the textual id used in the PowerSync protocol.
+  factory PendingStatementValue.id() = _PendingStmtValueId;
+
+  /// A value that is bound to the value of a column in a replace (`PUT`)
+  /// operation of the PowerSync protocol.
+  factory PendingStatementValue.column(String column) = _PendingStmtValueColumn;
+
+  dynamic toJson();
+}
+
+class _PendingStmtValueColumn implements PendingStatementValue {
+  final String column;
+  const _PendingStmtValueColumn(this.column);
+
+  @override
+  dynamic toJson() {
+    return {
+      'Column': column,
+    };
+  }
+}
+
+class _PendingStmtValueId implements PendingStatementValue {
+  const _PendingStmtValueId();
+
+  @override
+  dynamic toJson() {
+    return 'Id';
+  }
+}
+
 /// Type of column.
 enum ColumnType {
   /// TEXT column.