[WIP] Improve serialization also for legacy database versions

config/karma.config.js

@@ -63,6 +63,9 @@ module.exports = function (config) {
             test: /\.ts$/
           }
         ]
+      },
+      externals: {
+        fs: "fs"
       }
     },
     webpackMiddleware: {

config/nycrc.node.json

@@ -9,7 +9,8 @@
   "exclude": [
     "**/*.helper.[jt]s",
     "**/*.spec.[jt]s",
-    "**/spec/**/*.[jt]s"
+    "**/spec/**/*.[jt]s",
+    "**/lokijs/**/*.js"
   ],
   "report-dir": "./coverage/node/"
 }

packages/common/types.ts

@@ -1,10 +1,11 @@
 /**
  * @hidden
  */
-import { Loki } from "../loki/src/loki";
+import { Loki } from "../loki/src";
+import { Serialization } from "../loki/src/serialization/migration";
 
 export interface StorageAdapter {
-  loadDatabase(dbname: string): Promise<any>;
+  loadDatabase(dbname: string): Promise<string | Loki | Serialization.Serialized>;
 
   saveDatabase?(dbname: string, serialization: string): Promise<void>;
 
@@ -25,11 +26,10 @@ export type Doc<T extends object = object> = T & {
   };
 };
 
-export interface Dict<T> {
+export type Dict<T> = {
   [index: string]: T;
-
   [index: number]: T;
-}
+};
 
 
 

packages/fs-storage/spec/node/fs_storage.spec.ts

@@ -2,6 +2,9 @@
 import { Loki } from "../../../loki/src/loki";
 import { FSStorage } from "../../src/fs_storage";
 
+declare var require: (moduleId: string) => any;
+const loki = require("../../../lokijs/lokijs.js");
+
 describe("testing fs storage", function () {
 
   interface Name {
@@ -68,4 +71,23 @@ describe("testing fs storage", function () {
           });
       });
   });
+
+  it("from lokijs", (done) => {
+    const legacyDB = new loki("legacyDB", {persistenceMethod: "fs"});
+    const coll = legacyDB.addCollection("myColl");
+    coll.insert({name: "Hello World"});
+    legacyDB.saveDatabase(() => {
+      // Load with LokiDB.
+      const db = new Loki("legacyDB");
+      return db.initializePersistence()
+        .then(() => {
+          return db.loadDatabase();
+        }).then(() => {
+          expect(db.getCollection<Name>("myColl").find()[0].name).toEqual("Hello World");
+          return db.deleteDatabase();
+        }).then(() => {
+          done();
+        });
+    });
+  });
 });

packages/full-text-search/src/full_text_search.ts

@@ -5,6 +5,7 @@ import { PLUGINS } from "../../common/plugin";
 import { Query } from "./query_types";
 import { Scorer } from "./scorer";
 import { Analyzer } from "./analyzer/analyzer";
+import { Serialization } from "../../loki/src/serialization/migration";
 
 export class FullTextSearch {
   /// The id field of each document.
@@ -78,7 +79,7 @@ export class FullTextSearch {
     return this._idxSearcher.search(query);
   }
 
-  public toJSON(): FullTextSearch.Serialization {
+  public toJSON(): Serialization.FullTextSearch {
     let serialized = {id: this._id, ii: {}};
     let fieldNames = Object.keys(this._invIdxs);
     for (let i = 0; i < fieldNames.length; i++) {
@@ -88,7 +89,7 @@ export class FullTextSearch {
     return serialized;
   }
 
-  public static fromJSONObject(serialized: FullTextSearch.Serialization, analyzers: Dict<Analyzer> = {}): FullTextSearch {
+  public static fromJSONObject(serialized: Serialization.FullTextSearch, analyzers: Dict<Analyzer> = {}): FullTextSearch {
     let fts = new FullTextSearch([], serialized.id);
     let fieldNames = Object.keys(serialized.ii);
     for (let i = 0; i < fieldNames.length; i++) {
@@ -103,9 +104,4 @@ export namespace FullTextSearch {
   export interface FieldOptions extends InvertedIndex.FieldOptions {
     field: string;
   }
-
-  export interface Serialization {
-    id: string;
-    ii: Dict<InvertedIndex.Serialization>;
-  }
 }

packages/full-text-search/src/inverted_index.ts

@@ -1,4 +1,5 @@
 import { Analyzer, StandardAnalyzer, analyze } from "./analyzer/analyzer";
+import { Serialization } from "../../loki/src/serialization/migration";
 
 /**
  * Converts a string into an array of code points.
@@ -31,7 +32,7 @@ export class InvertedIndex {
   public docCount: number = 0;
   public docStore: Map<InvertedIndex.DocumentIndex, InvertedIndex.DocStore> = new Map();
   public totalFieldLength: number = 0;
-  public root: InvertedIndex.Index = new Map();
+  public root: InvertedIndex.Index = new Map() as InvertedIndex.Index;
 
   private _store: boolean;
   private _optimizeChanges: boolean;
@@ -219,20 +220,20 @@ export class InvertedIndex {
    * Serialize the inverted index.
    * @returns {{docStore: *, _fields: *, index: *}}
    */
-  public toJSON(): InvertedIndex.Serialization {
+  public toJSON(): Serialization.FullTextSearch.InvertedIndex {
     if (this._store) {
       return {
-        _store: true,
-        _optimizeChanges: this._optimizeChanges,
+        store: true,
+        optimizeChanges: this._optimizeChanges,
         docCount: this.docCount,
         docStore: [...this.docStore],
         totalFieldLength: this.totalFieldLength,
         root: InvertedIndex._serializeIndex(this.root)
       };
     }
     return {
-      _store: false,
-      _optimizeChanges: this._optimizeChanges,
+      store: false,
+      optimizeChanges: this._optimizeChanges,
     };
   }
 
@@ -241,14 +242,14 @@ export class InvertedIndex {
    * @param {{docStore: *, _fields: *, index: *}} serialized - The serialized inverted index.
    * @param {Analyzer} analyzer[undefined] - an analyzer
    */
-  public static fromJSONObject(serialized: InvertedIndex.Serialization, analyzer?: Analyzer): InvertedIndex {
+  public static fromJSONObject(serialized: Serialization.FullTextSearch.InvertedIndex, analyzer?: Analyzer): InvertedIndex {
     const invIdx = new InvertedIndex({
-      store: serialized._store,
-      optimizeChanges: serialized._optimizeChanges,
+      store: serialized.store,
+      optimizeChanges: serialized.optimizeChanges,
       analyzer: analyzer
     });
 
-    if (serialized._store) {
+    if (serialized.store) {
       invIdx.docCount = serialized.docCount;
       invIdx.docStore = new Map(serialized.docStore);
       invIdx.totalFieldLength = serialized.totalFieldLength;
@@ -262,8 +263,8 @@ export class InvertedIndex {
     return invIdx;
   }
 
-  private static _serializeIndex(idx: InvertedIndex.Index): InvertedIndex.SerializedIndex {
-    const serialized: InvertedIndex.SerializedIndex = {};
+  private static _serializeIndex(idx: InvertedIndex.Index): Serialization.FullTextSearch.Index {
+    const serialized: Serialization.FullTextSearch.Index = {};
     if (idx.dc !== undefined) {
       serialized.d = {df: idx.df, dc: [...idx.dc]};
     }
@@ -284,8 +285,8 @@ export class InvertedIndex {
     return serialized;
   }
 
-  private static _deserializeIndex(serialized: InvertedIndex.SerializedIndex): InvertedIndex.Index {
-    const idx: InvertedIndex.Index = new Map();
+  private static _deserializeIndex(serialized: Serialization.FullTextSearch.Index): InvertedIndex.Index {
+    const idx: InvertedIndex.Index = new Map() as InvertedIndex.Index;
 
     if (serialized.k !== undefined) {
       for (let i = 0; i < serialized.k.length; i++) {
@@ -374,31 +375,6 @@ export namespace InvertedIndex {
 
   export type IndexTerm = { index: Index, term: number[] };
 
-  export interface SerializedIndex {
-    d?: {
-      df: number;
-      dc: [DocumentIndex, number][]
-    };
-    k?: number[];
-    v?: SerializedIndex[];
-  }
-
-  export type Serialization = SpareSerialization | FullSerialization;
-
-  export type SpareSerialization = {
-    _store: false;
-    _optimizeChanges: boolean;
-  };
-
-  export type FullSerialization = {
-    _store: true;
-    _optimizeChanges: boolean;
-    docCount: number;
-    docStore: [DocumentIndex, DocStore][];
-    totalFieldLength: number;
-    root: SerializedIndex;
-  };
-
   export interface DocStore {
     fieldLength?: number;
     indexRef?: Index[];

packages/indexed-storage/spec/web/indexed_storage.spec.ts

@@ -2,6 +2,10 @@
 import { Loki } from "../../../loki/src/loki";
 import { IndexedStorage } from "../../src/indexed_storage";
 
+declare var require: (moduleId: string) => any;
+const loki = require("../../../lokijs/lokijs.js");
+const indexedAdapter = require("../../../lokijs/loki-indexed-adapter.js");
+
 describe("testing indexed storage", function () {
 
   interface Name {
@@ -68,4 +72,23 @@ describe("testing indexed storage", function () {
           });
       });
   });
+
+  it("from lokijs", (done) => {
+    const legacyDB = new loki("legacyDB", {adapter: new indexedAdapter()});
+    const coll = legacyDB.addCollection("myColl");
+    coll.insert({name: "Hello World"});
+    legacyDB.saveDatabase(() => {
+      // Load with LokiDB.
+      const db = new Loki("legacyDB");
+      return db.initializePersistence()
+        .then(() => {
+          return db.loadDatabase();
+        }).then(() => {
+          expect(db.getCollection<Name>("myColl").find()[0].name).toEqual("Hello World");
+          return db.deleteDatabase();
+        }).then(() => {
+          done();
+        });
+    });
+  });
 });

packages/indexed-storage/src/indexed_storage.ts

@@ -67,7 +67,7 @@ export class IndexedStorage implements StorageAdapter {
    * @param {string} dbname - the name of the database to retrieve.
    * @returns {Promise} a Promise that resolves after the database was loaded
    */
-  loadDatabase(dbname: string) {
+  loadDatabase(dbname: string): Promise<string> {
     const appName = this._appname;
     const adapter = this;
 

packages/local-storage/spec/web/local_storage.spec.ts

@@ -2,6 +2,9 @@
 import { Loki } from "../../../loki/src/loki";
 import { LocalStorage } from "../../src/local_storage";
 
+declare var require: (moduleId: string) => any;
+const loki = require("../../../lokijs/lokijs.js");
+
 describe("testing local storage", function () {
 
   interface Name {
@@ -68,4 +71,23 @@ describe("testing local storage", function () {
           });
       });
   });
+
+  it("from lokijs", (done) => {
+    const legacyDB = new loki("legacyDB", {persistenceMethod: "localStorage"});
+    const coll = legacyDB.addCollection("myColl");
+    coll.insert({name: "Hello World"});
+    legacyDB.saveDatabase(() => {
+      // Load with LokiDB.
+      const db = new Loki("legacyDB");
+      return db.initializePersistence()
+        .then(() => {
+          return db.loadDatabase();
+        }).then(() => {
+          expect(db.getCollection<Name>("myColl").find()[0].name).toEqual("Hello World");
+          return db.deleteDatabase();
+        }).then(() => {
+          done();
+        });
+    });
+  });
 });