From 9de57674f29d12d95718a66cc9438a85106d82aa Mon Sep 17 00:00:00 2001 From: Matt Avery Date: Fri, 29 Mar 2019 14:25:43 -0400 Subject: [PATCH 1/3] 64bit key support with bignums --- src/cursor.cpp | 5 +++ src/dbi.cpp | 6 +++ src/misc.cpp | 93 ++++++++++++++++++++++++++++++++++++++++------ src/node-lmdb.h | 9 +++++ test/index.test.js | 62 +++++++++++++++++++++++++++++++ 5 files changed, 163 insertions(+), 12 deletions(-) diff --git a/src/cursor.cpp b/src/cursor.cpp index 9926856723..0c747fac2f 100644 --- a/src/cursor.cpp +++ b/src/cursor.cpp @@ -67,6 +67,11 @@ NAN_METHOD(CursorWrap::ctor) { if (dw->keyType == NodeLmdbKeyType::Uint32Key && keyType != NodeLmdbKeyType::Uint32Key) { return Nan::ThrowError("You specified keyIsUint32 on the Dbi, so you can't use other key types with it."); } + #if NODE_LMDB_HAS_BIGINT + else if (dw->keyType == NodeLmdbKeyType::Uint64Key && keyType != NodeLmdbKeyType::Uint64Key) { + return Nan::ThrowError("You specified keyIsUint64 on the Dbi, so you can't use other key types with it."); + } + #endif // Open the cursor MDB_cursor *cursor; diff --git a/src/dbi.cpp b/src/dbi.cpp index 449ae7963a..b5ac002ab5 100644 --- a/src/dbi.cpp +++ b/src/dbi.cpp @@ -97,9 +97,15 @@ NAN_METHOD(DbiWrap::ctor) { return; } + #if NODE_LMDB_HAS_BIGINT if (keyType == NodeLmdbKeyType::Uint32Key) { flags |= MDB_INTEGERKEY; } + #else + if (keyType == NodeLmdbKeyType::Uint32Key || keyType == NodeLmdbKeyType::Uint64Key) { + flags |= MDB_INTEGERKEY; + } + #endif // Set flags for txn used to open database Local create = options->Get(Nan::New("create").ToLocalChecked()); diff --git a/src/misc.cpp b/src/misc.cpp index 850eb6d82a..a5feae57fd 100644 --- a/src/misc.cpp +++ b/src/misc.cpp @@ -26,6 +26,30 @@ #include static thread_local uint32_t currentUint32Key = 0; +#ifdef NODE_LMDB_HAS_BIGINT +static thread_local uint64_t currentUint64Key = 0; + +inline bool get_uint64_from_bigint(const Local &val, uint64_t *ret = nullptr) { + bool lossless = true; + + Local context = v8::Isolate::GetCurrent()->GetCurrentContext(); + Local bigint_val; + if (!val->ToBigInt(context).ToLocal(&bigint_val)) { + return false; + } + if(ret) { + *ret = bigint_val->Uint64Value(&lossless); + } else { + bigint_val->Uint64Value(&lossless); + } + return lossless; +} + +inline Local uint64_blob_to_bigint(void* data) { + //TODO use nan when it supports bigint + return v8::BigInt::NewFromUnsigned(v8::Isolate::GetCurrent(), *((uint64_t*)data)); +} +#endif void setupExportMisc(Handle exports) { Local versionObj = Nan::New(); @@ -62,32 +86,49 @@ NodeLmdbKeyType keyTypeFromOptions(const Local &val, NodeLmdbKeyType defa int keyIsUint32 = 0; int keyIsBuffer = 0; int keyIsString = 0; + #if NODE_LMDB_HAS_BIGINT + int keyIsUint64 = 0; + #endif setFlagFromValue(&keyIsUint32, 1, "keyIsUint32", false, obj); setFlagFromValue(&keyIsString, 1, "keyIsString", false, obj); setFlagFromValue(&keyIsBuffer, 1, "keyIsBuffer", false, obj); - - const char *keySpecificationErrorText = "You can't specify multiple key types at once. Either set keyIsUint32, or keyIsBuffer or keyIsString (default)."; + #if NODE_LMDB_HAS_BIGINT + setFlagFromValue(&keyIsUint64, 1, "keyIsUint64", false, obj); + #endif + + int key_spec_count = keyIsUint32 + keyIsBuffer + keyIsString; + #if NODE_LMDB_HAS_BIGINT + key_spec_count += keyIsUint64; + #endif + + if (key_spec_count == 0) { + return keyType; + } else if (key_spec_count != 1) { + #if NODE_LMDB_HAS_BIGINT + const char *keySpecificationErrorText = "You can't specify multiple key types at once. Either set keyIsUint32, or keyIsUint64, or keyIsBuffer, or keyIsString (default)."; + #else + const char *keySpecificationErrorText = "You can't specify multiple key types at once. Either set keyIsUint32, or keyIsBuffer, or keyIsString (default)."; + #endif + Nan::ThrowError(keySpecificationErrorText); + return NodeLmdbKeyType::InvalidKey; + } + if (keyIsUint32) { keyType = NodeLmdbKeyType::Uint32Key; - - if (keyIsBuffer || keyIsString) { - Nan::ThrowError(keySpecificationErrorText); - return NodeLmdbKeyType::InvalidKey; - } } else if (keyIsBuffer) { keyType = NodeLmdbKeyType::BinaryKey; - - if (keyIsUint32 || keyIsString) { - Nan::ThrowError(keySpecificationErrorText); - return NodeLmdbKeyType::InvalidKey; - } } else if (keyIsString) { keyType = NodeLmdbKeyType::StringKey; } + #if NODE_LMDB_HAS_BIGINT + else if (keyIsUint64) { + keyType = NodeLmdbKeyType::Uint64Key; + } + #endif return keyType; } @@ -99,6 +140,11 @@ NodeLmdbKeyType inferKeyType(const Local &val) { if (val->IsUint32()) { return NodeLmdbKeyType::Uint32Key; } + #if NODE_LMDB_HAS_BIGINT + if (val->IsBigInt() && get_uint64_from_bigint(val)) { + return NodeLmdbKeyType::Uint64Key; + } + #endif if (node::Buffer::HasInstance(val)) { return NodeLmdbKeyType::BinaryKey; } @@ -122,6 +168,12 @@ NodeLmdbKeyType inferAndValidateKeyType(const Local &key, const Local &val, MDB_val &key, NodeLmdbKeyT return nullptr; } + + else if (keyType == NodeLmdbKeyType::Uint64Key) { + if (!val->IsBigInt() || !get_uint64_from_bigint(val, ¤tUint64Key)) { + Nan::ThrowError("Invalid key. Should be an unsigned 64-bit integer. (Specified with env.openDbi)"); + return nullptr; + } + + isValid = true; + key.mv_size = sizeof(uint64_t); + key.mv_data = ¤tUint64Key; + + return nullptr; + } else if (keyType == NodeLmdbKeyType::InvalidKey) { Nan::ThrowError("Invalid key type. This might be a bug in node-lmdb."); } @@ -185,6 +250,10 @@ Local keyToHandle(MDB_val &key, NodeLmdbKeyType keyType) { return valToBinary(key); case NodeLmdbKeyType::StringKey: return valToString(key); + #if NODE_LMDB_HAS_BIGINT + case NodeLmdbKeyType::Uint64Key: + return uint64_blob_to_bigint(key.mv_data); + #endif default: Nan::ThrowError("Unknown key type. This is a bug in node-lmdb."); return Nan::Undefined(); diff --git a/src/node-lmdb.h b/src/node-lmdb.h index 1a92deb71c..d28135e1e5 100644 --- a/src/node-lmdb.h +++ b/src/node-lmdb.h @@ -33,6 +33,9 @@ #include #include "lmdb.h" +//Bigint added in version 10.4 +#define NODE_LMDB_HAS_BIGINT (NODE_MAJOR_VERSION > 10 || NODE_MAJOR_VERSION == 10 && NODE_MINOR_VERSION >= 4) + using namespace v8; using namespace node; @@ -53,6 +56,11 @@ enum class NodeLmdbKeyType { // LMDB default key format - Appears to V8 as node::Buffer BinaryKey = 3, +#if NODE_LMDB_HAS_BIGINT + // LMDB fixed size integer key with 32 bit keys - Appearts to V8 as a Bigint which + // is then attempted to be interpreted as aUint64 + Uint64Key = 4, +#endif }; // Exports misc stuff to the module @@ -192,6 +200,7 @@ class EnvWrap : public Nan::ObjectWrap { * name: the name of the database (or null to use the unnamed database) * create: if true, the database will be created if it doesn't exist * keyIsUint32: if true, keys are treated as 32-bit unsigned integers + * keyIsUint64: (only in node 10.4.0 or later) if true, keys are treated as 64-bit unsigned integers * dupSort: if true, the database can hold multiple items with the same key * reverseKey: keys are strings to be compared in reverse order * dupFixed: if dupSort is true, indicates that the data items are all the same size diff --git a/test/index.test.js b/test/index.test.js index e05dedc35e..62a7b27cf0 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -1245,5 +1245,67 @@ describe('Node.js LMDB Bindings', function() { }, 100); }); }); + describe('Multiple transactions (64-bit keys)', function() { + this.timeout(10000); + var env; + var dbi; + before(function() { + env = new lmdb.Env(); + env.open({ + path: testDirPath, + maxDbs: 10 + }); + dbi = env.openDbi({ + name: 'mydb6', + create: true, + keyIsUint64: true + }); + var txn = env.beginTxn(); + //9223372036854776000n is 2^63 and rounded in a regular js Number object: + //ie: + //>9223372036854776001 === 9223372036854776002 + //true + //>9223372036854776001n === 9223372036854776002n + //false + txn.putString(dbi, 9223372036854776001n, 'Hello9223372036854776001'); + txn.putString(dbi, 9223372036854776002n, 'Hello9223372036854776002'); + txn.commit(); + }); + after(function() { + dbi.close(); + env.close(); + }); + it('readonly transaction should not see uncommited changes', function() { + var readTxn = env.beginTxn({readOnly: true}); + var data = readTxn.getString(dbi, 9223372036854776001n); + should.equal(data, 'Hello9223372036854776001'); + + var writeTxn = env.beginTxn(); + writeTxn.putString(dbi, 9223372036854776001n, 'Ha ha ha'); + + var data2 = writeTxn.getString(dbi, 9223372036854776001n); + data2.should.equal('Ha ha ha'); + + var data3 = readTxn.getString(dbi, 9223372036854776001n); + should.equal(data3, 'Hello9223372036854776001'); + + writeTxn.commit(); + var data4 = readTxn.getString(dbi, 9223372036854776001n); + should.equal(data4, 'Hello9223372036854776001'); + + readTxn.reset(); + readTxn.renew(); + var data5 = readTxn.getString(dbi, 9223372036854776001n); + should.equal(data5, 'Ha ha ha'); + readTxn.abort(); + }); + it('readonly transaction will throw if tries to write', function() { + var readTxn = env.beginTxn({readOnly: true}); + (function() { + readTxn.putString(dbi, 9223372036854776002n, 'hööhh'); + }).should.throw('Permission denied'); + readTxn.abort(); + }); + }); }); From 485ff813f5101a5105257a5c79552a1c93de7933 Mon Sep 17 00:00:00 2001 From: Matt Avery Date: Fri, 29 Mar 2019 15:36:38 -0400 Subject: [PATCH 2/3] #define fixups --- src/dbi.cpp | 4 ++-- src/misc.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dbi.cpp b/src/dbi.cpp index b5ac002ab5..bbfa2c2e59 100644 --- a/src/dbi.cpp +++ b/src/dbi.cpp @@ -98,11 +98,11 @@ NAN_METHOD(DbiWrap::ctor) { } #if NODE_LMDB_HAS_BIGINT - if (keyType == NodeLmdbKeyType::Uint32Key) { + if (keyType == NodeLmdbKeyType::Uint32Key || keyType == NodeLmdbKeyType::Uint64Key) { flags |= MDB_INTEGERKEY; } #else - if (keyType == NodeLmdbKeyType::Uint32Key || keyType == NodeLmdbKeyType::Uint64Key) { + if (keyType == NodeLmdbKeyType::Uint32Key) { flags |= MDB_INTEGERKEY; } #endif diff --git a/src/misc.cpp b/src/misc.cpp index a5feae57fd..8095e16c61 100644 --- a/src/misc.cpp +++ b/src/misc.cpp @@ -26,7 +26,7 @@ #include static thread_local uint32_t currentUint32Key = 0; -#ifdef NODE_LMDB_HAS_BIGINT +#if NODE_LMDB_HAS_BIGINT static thread_local uint64_t currentUint64Key = 0; inline bool get_uint64_from_bigint(const Local &val, uint64_t *ret = nullptr) { From 47962b12ac9d36bf4d50f220710e285a7aff944c Mon Sep 17 00:00:00 2001 From: Matt Avery Date: Fri, 29 Mar 2019 15:40:18 -0400 Subject: [PATCH 3/3] missed a #define. final commit for this PR --- src/misc.cpp | 3 ++- src/node-lmdb.h | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/misc.cpp b/src/misc.cpp index 8095e16c61..7ba0e31c32 100644 --- a/src/misc.cpp +++ b/src/misc.cpp @@ -219,7 +219,7 @@ argtokey_callback_t argToKey(const Local &val, MDB_val &key, NodeLmdbKeyT return nullptr; } - + #if NODE_LMDB_HAS_BIGINT else if (keyType == NodeLmdbKeyType::Uint64Key) { if (!val->IsBigInt() || !get_uint64_from_bigint(val, ¤tUint64Key)) { Nan::ThrowError("Invalid key. Should be an unsigned 64-bit integer. (Specified with env.openDbi)"); @@ -232,6 +232,7 @@ argtokey_callback_t argToKey(const Local &val, MDB_val &key, NodeLmdbKeyT return nullptr; } + #endif else if (keyType == NodeLmdbKeyType::InvalidKey) { Nan::ThrowError("Invalid key type. This might be a bug in node-lmdb."); } diff --git a/src/node-lmdb.h b/src/node-lmdb.h index d28135e1e5..edb4c2ab19 100644 --- a/src/node-lmdb.h +++ b/src/node-lmdb.h @@ -34,7 +34,9 @@ #include "lmdb.h" //Bigint added in version 10.4 -#define NODE_LMDB_HAS_BIGINT (NODE_MAJOR_VERSION > 10 || NODE_MAJOR_VERSION == 10 && NODE_MINOR_VERSION >= 4) +#ifndef NODE_LMDB_HAS_BIGINT + #define NODE_LMDB_HAS_BIGINT (NODE_MAJOR_VERSION > 10 || NODE_MAJOR_VERSION == 10 && NODE_MINOR_VERSION >= 4) +#endif using namespace v8; using namespace node;