From c7e80662d2e1d539854fa343fc6cab807570780e Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Wed, 4 Dec 2024 09:36:57 -0500 Subject: [PATCH 01/16] Fix some incorrect allocator types mismatch. --- .../chain/include/eosio/chain/protocol_state_object.hpp | 9 +++++---- libraries/chainbase | 2 +- unittests/test_chainbase_types.cpp | 7 ++++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/libraries/chain/include/eosio/chain/protocol_state_object.hpp b/libraries/chain/include/eosio/chain/protocol_state_object.hpp index c2ffae1b76..89dcf8d714 100644 --- a/libraries/chain/include/eosio/chain/protocol_state_object.hpp +++ b/libraries/chain/include/eosio/chain/protocol_state_object.hpp @@ -17,10 +17,11 @@ namespace eosio { namespace chain { class protocol_state_object : public chainbase::object { public: - template - protocol_state_object(Constructor&& c, chainbase::constructor_tag) : - id(0), - whitelisted_intrinsics(*activated_protocol_features.get_allocator(this)) { + template + protocol_state_object(Constructor&& c, chainbase::constructor_tag) + : id(0) + , whitelisted_intrinsics( + chainbase::make_allocator(&whitelisted_intrinsics)) { c(*this); } diff --git a/libraries/chainbase b/libraries/chainbase index 8992c78bad..9614c9de2f 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 8992c78badf70b9bee716da2461e7e3278a3d5d6 +Subproject commit 9614c9de2fe281af2188c7459b994ca110d5d1f9 diff --git a/unittests/test_chainbase_types.cpp b/unittests/test_chainbase_types.cpp index ff4760344f..e69d25de36 100644 --- a/unittests/test_chainbase_types.cpp +++ b/unittests/test_chainbase_types.cpp @@ -73,7 +73,8 @@ BOOST_AUTO_TEST_CASE(chainbase_type_segment_alloc) { fs::path temp = temp_dir.path() / "pinnable_mapped_file_101"; pinnable_mapped_file pmf(temp, true, 1024 * 1024, false, pinnable_mapped_file::map_mode::mapped); - chainbase::allocator alloc(pmf.get_segment_manager()); + auto seg_mgr = pmf.get_segment_manager(); + auto alloc = chainbase::make_allocator(seg_mgr); bip_vector> v(alloc); bip_vector> v2(alloc); @@ -82,6 +83,6 @@ BOOST_AUTO_TEST_CASE(chainbase_type_segment_alloc) { auto a2 = v2[1].authors[0].get_allocator(); // check that objects inside the vectors are allocated within the pinnable_mapped_file segment - BOOST_REQUIRE(a && (chainbase::allocator)(*a) == alloc); - BOOST_REQUIRE(a2 && (chainbase::allocator)(*a2) == alloc); + BOOST_REQUIRE(a && *a == chainbase::make_allocator(seg_mgr)); + BOOST_REQUIRE(a2 && *a2 == chainbase::make_allocator(seg_mgr)); } From e1438ca6c83e0398a079d229b7b59f1a85559ebd Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 6 Dec 2024 17:34:30 -0500 Subject: [PATCH 02/16] Log used db size after loading the snapshot. --- libraries/chain/controller.cpp | 5 ++++- libraries/chainbase | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 7fc8cb387d..c83bf8289c 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1926,7 +1926,10 @@ struct controller_impl { } }); auto snapshot_load_time = (fc::time_point::now() - snapshot_load_start_time).to_seconds(); - ilog( "Finished initialization from snapshot (snapshot load time was ${t}s)", ("t", snapshot_load_time) ); + auto db_size = db.get_segment_manager()->get_size(); + auto free_size = db.get_segment_manager()->get_free_memory(); + + ilog( "Finished initialization from snapshot (snapshot load time was ${t}s, db size used is ${s} bytes)", ("t", snapshot_load_time)("s", db_size - free_size) ); } catch (boost::interprocess::bad_alloc& e) { elog( "Failed initialization from snapshot - db storage not configured to have enough storage for the provided snapshot, please increase and retry snapshot" ); shutdown(); diff --git a/libraries/chainbase b/libraries/chainbase index 9614c9de2f..5bedb5ceff 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 9614c9de2fe281af2188c7459b994ca110d5d1f9 +Subproject commit 5bedb5ceff2b398fc556b507b8d2c328b3fbd526 From 3f6b52ccc2b1b438b452245b9c6a43520729d52b Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Wed, 11 Dec 2024 10:20:51 -0500 Subject: [PATCH 03/16] Add call to preallocate for large tables. --- libraries/chain/controller.cpp | 2 ++ libraries/chain/include/eosio/chain/database_utils.hpp | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index c83bf8289c..f4ff326ca9 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1925,6 +1925,7 @@ struct controller_impl { } } }); + auto snapshot_load_time = (fc::time_point::now() - snapshot_load_start_time).to_seconds(); auto db_size = db.get_segment_manager()->get_size(); auto free_size = db.get_segment_manager()->get_free_memory(); @@ -2219,6 +2220,7 @@ struct controller_impl { section.read_row(rows_for_this_tid, db); read_row_count.fetch_add(2u, std::memory_order_relaxed); + utils_t::preallocate(db, rows_for_this_tid.value); for(size_t idx = 0; idx < rows_for_this_tid.value; idx++) { utils_t::create(db, [this, §ion, &more, &t_id](auto& row) { row.t_id = t_id; diff --git a/libraries/chain/include/eosio/chain/database_utils.hpp b/libraries/chain/include/eosio/chain/database_utils.hpp index 24490a65c9..5e196c8495 100644 --- a/libraries/chain/include/eosio/chain/database_utils.hpp +++ b/libraries/chain/include/eosio/chain/database_utils.hpp @@ -60,6 +60,10 @@ namespace eosio::chain { static void create( chainbase::database& db, F cons ) { db.create(cons); } + + static void preallocate( chainbase::database& db, size_t num ) { + db.preallocate(num); + } }; template From d24f759badcc975b01ca65a5cd3d019dfacadd91 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 13 Dec 2024 18:37:36 -0500 Subject: [PATCH 04/16] Disable preallocation which does not seem to make a significant difference. --- libraries/chain/controller.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index f4ff326ca9..b141966a49 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -2220,7 +2220,7 @@ struct controller_impl { section.read_row(rows_for_this_tid, db); read_row_count.fetch_add(2u, std::memory_order_relaxed); - utils_t::preallocate(db, rows_for_this_tid.value); + // utils_t::preallocate(db, rows_for_this_tid.value); for(size_t idx = 0; idx < rows_for_this_tid.value; idx++) { utils_t::create(db, [this, §ion, &more, &t_id](auto& row) { row.t_id = t_id; From a1452c584c4079d6c4baf6b4d357d09c8db2d1a1 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 13 Dec 2024 18:38:28 -0500 Subject: [PATCH 05/16] Update to latest chainbase tip (of branch `gh_1049`) --- libraries/chainbase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chainbase b/libraries/chainbase index 5bedb5ceff..d127509f0b 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 5bedb5ceff2b398fc556b507b8d2c328b3fbd526 +Subproject commit d127509f0b6cb7ee4247f22fd2bc4bfb6013009d From bacbc3edb96ff3474a12dc4d7b7d99075f540600 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Sun, 15 Dec 2024 20:52:12 -0500 Subject: [PATCH 06/16] Update chainbase to branch tip. --- libraries/chainbase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chainbase b/libraries/chainbase index d127509f0b..54950d3554 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit d127509f0b6cb7ee4247f22fd2bc4bfb6013009d +Subproject commit 54950d355492a7f22967037164174a7a56a243e5 From 726553b341e1409945ecfdf52093dcadc17ae5ad Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Sun, 15 Dec 2024 20:55:31 -0500 Subject: [PATCH 07/16] Update chainbase to branch tip. --- libraries/chainbase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chainbase b/libraries/chainbase index 54950d3554..27a561dc02 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 54950d355492a7f22967037164174a7a56a243e5 +Subproject commit 27a561dc0218d6eef8871be6709d3d69063a70b1 From ee5964bcea3e9f0123d09f4f99361238d1cbcef7 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Mon, 16 Dec 2024 11:16:40 -0500 Subject: [PATCH 08/16] Update to chainbase branch tip. --- libraries/chainbase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chainbase b/libraries/chainbase index 27a561dc02..a0797d2be1 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 27a561dc0218d6eef8871be6709d3d69063a70b1 +Subproject commit a0797d2be1f4021dee2fdfed4e622ad9cc292dfe From 0d9a8e4008c59ee7c05b9efb3faa45fdf25bc3e5 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Tue, 17 Dec 2024 14:20:45 -0500 Subject: [PATCH 09/16] Update chainbase to branch tip. --- libraries/chainbase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chainbase b/libraries/chainbase index a0797d2be1..91df26d077 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit a0797d2be1f4021dee2fdfed4e622ad9cc292dfe +Subproject commit 91df26d07752ba96f25a829650fc367b4bb3e5c8 From 5b4c18198bdc837ccf02a669213b78c96e66ddef Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Wed, 4 Dec 2024 09:36:57 -0500 Subject: [PATCH 10/16] Fix some incorrect allocator types mismatch. --- .../chain/include/eosio/chain/protocol_state_object.hpp | 9 +++++---- libraries/chainbase | 2 +- unittests/test_chainbase_types.cpp | 7 ++++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/libraries/chain/include/eosio/chain/protocol_state_object.hpp b/libraries/chain/include/eosio/chain/protocol_state_object.hpp index c2ffae1b76..89dcf8d714 100644 --- a/libraries/chain/include/eosio/chain/protocol_state_object.hpp +++ b/libraries/chain/include/eosio/chain/protocol_state_object.hpp @@ -17,10 +17,11 @@ namespace eosio { namespace chain { class protocol_state_object : public chainbase::object { public: - template - protocol_state_object(Constructor&& c, chainbase::constructor_tag) : - id(0), - whitelisted_intrinsics(*activated_protocol_features.get_allocator(this)) { + template + protocol_state_object(Constructor&& c, chainbase::constructor_tag) + : id(0) + , whitelisted_intrinsics( + chainbase::make_allocator(&whitelisted_intrinsics)) { c(*this); } diff --git a/libraries/chainbase b/libraries/chainbase index d7ba67b33b..6b7b0eb0f3 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit d7ba67b33b3c5adb8210d1192a0ecec69d0c0699 +Subproject commit 6b7b0eb0f3eb74a3c6e8849dd1d59dd20b3b8f05 diff --git a/unittests/test_chainbase_types.cpp b/unittests/test_chainbase_types.cpp index ff4760344f..e69d25de36 100644 --- a/unittests/test_chainbase_types.cpp +++ b/unittests/test_chainbase_types.cpp @@ -73,7 +73,8 @@ BOOST_AUTO_TEST_CASE(chainbase_type_segment_alloc) { fs::path temp = temp_dir.path() / "pinnable_mapped_file_101"; pinnable_mapped_file pmf(temp, true, 1024 * 1024, false, pinnable_mapped_file::map_mode::mapped); - chainbase::allocator alloc(pmf.get_segment_manager()); + auto seg_mgr = pmf.get_segment_manager(); + auto alloc = chainbase::make_allocator(seg_mgr); bip_vector> v(alloc); bip_vector> v2(alloc); @@ -82,6 +83,6 @@ BOOST_AUTO_TEST_CASE(chainbase_type_segment_alloc) { auto a2 = v2[1].authors[0].get_allocator(); // check that objects inside the vectors are allocated within the pinnable_mapped_file segment - BOOST_REQUIRE(a && (chainbase::allocator)(*a) == alloc); - BOOST_REQUIRE(a2 && (chainbase::allocator)(*a2) == alloc); + BOOST_REQUIRE(a && *a == chainbase::make_allocator(seg_mgr)); + BOOST_REQUIRE(a2 && *a2 == chainbase::make_allocator(seg_mgr)); } From 910d009e509f6c914ea6c384f498763220086f34 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 6 Dec 2024 17:34:30 -0500 Subject: [PATCH 11/16] Log used db size after loading the snapshot. --- libraries/chain/controller.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 162105e6cf..6012385322 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1984,7 +1984,10 @@ struct controller_impl { } }); auto snapshot_load_time = (fc::time_point::now() - snapshot_load_start_time).to_seconds(); - ilog( "Finished initialization from snapshot (snapshot load time was ${t}s)", ("t", snapshot_load_time) ); + auto db_size = db.get_segment_manager()->get_size(); + auto free_size = db.get_segment_manager()->get_free_memory(); + + ilog( "Finished initialization from snapshot (snapshot load time was ${t}s, db size used is ${s} bytes)", ("t", snapshot_load_time)("s", db_size - free_size) ); } catch (boost::interprocess::bad_alloc& e) { elog( "Failed initialization from snapshot - db storage not configured to have enough storage for the provided snapshot, please increase and retry snapshot" ); shutdown(); From 2ade1f8bd1be46dbc37b64f5b5b19763865a7be3 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Wed, 11 Dec 2024 10:20:51 -0500 Subject: [PATCH 12/16] Add call to preallocate for large tables. --- libraries/chain/controller.cpp | 2 ++ libraries/chain/include/eosio/chain/database_utils.hpp | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 6012385322..d4eed5ea37 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1983,6 +1983,7 @@ struct controller_impl { } } }); + auto snapshot_load_time = (fc::time_point::now() - snapshot_load_start_time).to_seconds(); auto db_size = db.get_segment_manager()->get_size(); auto free_size = db.get_segment_manager()->get_free_memory(); @@ -2242,6 +2243,7 @@ struct controller_impl { section.read_row(rows_for_this_tid, db); read_row_count.fetch_add(2u, std::memory_order_relaxed); + utils_t::preallocate(db, rows_for_this_tid.value); for(size_t idx = 0; idx < rows_for_this_tid.value; idx++) { utils_t::create(db, [this, §ion, &more, &t_id](auto& row) { row.t_id = t_id; diff --git a/libraries/chain/include/eosio/chain/database_utils.hpp b/libraries/chain/include/eosio/chain/database_utils.hpp index 24490a65c9..5e196c8495 100644 --- a/libraries/chain/include/eosio/chain/database_utils.hpp +++ b/libraries/chain/include/eosio/chain/database_utils.hpp @@ -60,6 +60,10 @@ namespace eosio::chain { static void create( chainbase::database& db, F cons ) { db.create(cons); } + + static void preallocate( chainbase::database& db, size_t num ) { + db.preallocate(num); + } }; template From a936c1e0355ec82964d26fb201f3fd14587eb7f4 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 13 Dec 2024 18:37:36 -0500 Subject: [PATCH 13/16] Disable preallocation which does not seem to make a significant difference. --- libraries/chain/controller.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index d4eed5ea37..496dcd3a6b 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -2243,7 +2243,7 @@ struct controller_impl { section.read_row(rows_for_this_tid, db); read_row_count.fetch_add(2u, std::memory_order_relaxed); - utils_t::preallocate(db, rows_for_this_tid.value); + // utils_t::preallocate(db, rows_for_this_tid.value); for(size_t idx = 0; idx < rows_for_this_tid.value; idx++) { utils_t::create(db, [this, §ion, &more, &t_id](auto& row) { row.t_id = t_id; From 484a8d73e0ece94e0f318efe677c26d1fcf15703 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Wed, 23 Apr 2025 17:16:20 -0400 Subject: [PATCH 14/16] Remove commented-out line. --- libraries/chain/controller.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 496dcd3a6b..12629cac5e 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -2243,7 +2243,6 @@ struct controller_impl { section.read_row(rows_for_this_tid, db); read_row_count.fetch_add(2u, std::memory_order_relaxed); - // utils_t::preallocate(db, rows_for_this_tid.value); for(size_t idx = 0; idx < rows_for_this_tid.value; idx++) { utils_t::create(db, [this, §ion, &more, &t_id](auto& row) { row.t_id = t_id; From f80dbd5abb8d1d6d87d229ec9730aa0b806dd3a9 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Thu, 24 Apr 2025 10:25:10 -0400 Subject: [PATCH 15/16] Update chainbase to branch tip. --- libraries/chainbase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chainbase b/libraries/chainbase index 6b7b0eb0f3..9a558e1631 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 6b7b0eb0f3eb74a3c6e8849dd1d59dd20b3b8f05 +Subproject commit 9a558e1631f291cb9a40293010fb4e414ee03008 From 87c01b47c798c48c45b093a1a1b5f4520cd1827d Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Mon, 28 Apr 2025 14:54:07 -0400 Subject: [PATCH 16/16] Move changes from `chainbase` PR #54. --- .../chaindb/include/chainbase/chainbase.hpp | 10 + .../chainbase/chainbase_node_allocator.hpp | 67 ++++-- .../chaindb/include/chainbase/environment.hpp | 15 +- .../chainbase/pinnable_mapped_file.hpp | 50 ++++- .../include/chainbase/shared_cow_string.hpp | 2 +- .../include/chainbase/shared_cow_vector.hpp | 2 +- .../chainbase/small_size_allocator.hpp | 207 ++++++++++++++++++ .../chaindb/include/chainbase/undo_index.hpp | 4 + .../chaindb/src/pinnable_mapped_file.cpp | 19 +- libraries/chaindb/test/test.cpp | 40 ++-- 10 files changed, 364 insertions(+), 52 deletions(-) create mode 100644 libraries/chaindb/include/chainbase/small_size_allocator.hpp diff --git a/libraries/chaindb/include/chainbase/chainbase.hpp b/libraries/chaindb/include/chainbase/chainbase.hpp index 0e29d1eaa2..315987ed48 100644 --- a/libraries/chaindb/include/chainbase/chainbase.hpp +++ b/libraries/chaindb/include/chainbase/chainbase.hpp @@ -509,6 +509,16 @@ namespace chainbase { return get_mutable_index().remove( obj ); } + template + void preallocate( size_t num ) + { + if ( _read_only_mode ) { + BOOST_THROW_EXCEPTION( std::logic_error( "attempting to preallocate in read-only mode" ) ); + } + typedef typename get_index_type::type index_type; + get_mutable_index().preallocate( num ); + } + template const ObjectType& create( Constructor&& con ) { diff --git a/libraries/chaindb/include/chainbase/chainbase_node_allocator.hpp b/libraries/chaindb/include/chainbase/chainbase_node_allocator.hpp index 2c0575ead3..f51d667582 100644 --- a/libraries/chaindb/include/chainbase/chainbase_node_allocator.hpp +++ b/libraries/chaindb/include/chainbase/chainbase_node_allocator.hpp @@ -14,57 +14,82 @@ namespace chainbase { public: using value_type = T; using pointer = bip::offset_ptr; - chainbase_node_allocator(segment_manager* manager) : _manager{manager} {} - chainbase_node_allocator(const chainbase_node_allocator& other) : _manager(other._manager) {} + + chainbase_node_allocator(segment_manager* manager) : _manager{manager} { + _ss_alloc = pinnable_mapped_file::get_small_size_allocator((std::byte*)manager); + } + + chainbase_node_allocator(const chainbase_node_allocator& other) : chainbase_node_allocator(&*other._manager) {} + template - chainbase_node_allocator(const chainbase_node_allocator& other) : _manager(other._manager) {} + chainbase_node_allocator(const chainbase_node_allocator& other) : chainbase_node_allocator(&*other._manager) {} + pointer allocate(std::size_t num) { if (num == 1) { - if (_freelist == nullptr) { - get_some(); + if (_block_start == _block_end && _freelist == nullptr) { + get_some(_allocation_batch_size); } + if (_block_start < _block_end) { + pointer result = pointer{static_cast(static_cast(_block_start.get()))}; + _block_start += sizeof(T); + return result; + } + assert(_freelist != nullptr); list_item* result = &*_freelist; _freelist = _freelist->_next; result->~list_item(); --_freelist_size; return pointer{(T*)result}; } else { - return pointer{(T*)_manager->allocate(num*sizeof(T))}; + return pointer{(T*)&*_ss_alloc->allocate(num*sizeof(T))}; } } + void deallocate(const pointer& p, std::size_t num) { if (num == 1) { _freelist = new (&*p) list_item{_freelist}; ++_freelist_size; } else { - _manager->deallocate(&*p); + _ss_alloc->deallocate(ss_allocator_t::pointer((char*)&*p), num*sizeof(T)); } } + + void preallocate(std::size_t num) { + if (num >= 2 * _allocation_batch_size) + get_some((num + 7) & ~7); + } + bool operator==(const chainbase_node_allocator& other) const { return this == &other; } bool operator!=(const chainbase_node_allocator& other) const { return this != &other; } segment_manager* get_segment_manager() const { return _manager.get(); } - size_t freelist_memory_usage() const { return _freelist_size * sizeof(T); } + size_t freelist_memory_usage() const { return _freelist_size * sizeof(T) + (_block_end - _block_start); } + private: template friend class chainbase_node_allocator; - void get_some() { + + void get_some(size_t num_to_alloc) { static_assert(sizeof(T) >= sizeof(list_item), "Too small for free list"); static_assert(sizeof(T) % alignof(list_item) == 0, "Bad alignment for free list"); - const unsigned allocation_batch_size = 64; - char* result = (char*)_manager->allocate(sizeof(T) * allocation_batch_size); - _freelist_size += allocation_batch_size; - _freelist = bip::offset_ptr{(list_item*)result}; - for(unsigned i = 0; i < allocation_batch_size-1; ++i) { - char* next = result + sizeof(T); - new(result) list_item{bip::offset_ptr{(list_item*)next}}; - result = next; - } - new(result) list_item{nullptr}; + + _block_start = static_cast(_manager->allocate(sizeof(T) * num_to_alloc)); + _block_end = _block_start + sizeof(T) * num_to_alloc; + + if (_allocation_batch_size < max_allocation_batch_size) + _allocation_batch_size *= 2; } + struct list_item { bip::offset_ptr _next; }; + + static constexpr size_t max_allocation_batch_size = 512; + + bip::offset_ptr _block_start; + bip::offset_ptr _block_end; + bip::offset_ptr _freelist{}; + bip::offset_ptr _ss_alloc; bip::offset_ptr _manager; - bip::offset_ptr _freelist{}; - size_t _freelist_size = 0; + size_t _allocation_batch_size = 32; + size_t _freelist_size = 0; }; } // namepsace chainbase diff --git a/libraries/chaindb/include/chainbase/environment.hpp b/libraries/chaindb/include/chainbase/environment.hpp index 322f9be309..cc116aeee5 100644 --- a/libraries/chaindb/include/chainbase/environment.hpp +++ b/libraries/chaindb/include/chainbase/environment.hpp @@ -5,9 +5,13 @@ namespace chainbase { constexpr size_t header_size = 1024; + // `CHAINB01` reflects changes since `EOSIODB3`. +// `CHAINB02` adds the small size allocator // Spring 1.0 is compatible with `CHAINB01`. -constexpr uint64_t header_id = 0x3130424e49414843ULL; //"CHAINB01" little endian +// Spring 2.0 is compatible with `CHAINB02`. +// --------------------------------------------- +constexpr uint64_t header_id = 0x3230424e49414843ULL; //"CHAINB02" little endian struct environment { environment() { @@ -67,10 +71,11 @@ struct environment { } __attribute__ ((packed)); struct db_header { - uint64_t id = header_id; - bool dirty = false; - environment dbenviron; -} __attribute__ ((packed)); + uint64_t id = header_id; + bool dirty = false; + bip::offset_ptr small_size_allocator; + environment dbenviron; +}; constexpr size_t header_dirty_bit_offset = offsetof(db_header, dirty); diff --git a/libraries/chaindb/include/chainbase/pinnable_mapped_file.hpp b/libraries/chaindb/include/chainbase/pinnable_mapped_file.hpp index 7d3172c48d..2e61ea6f65 100644 --- a/libraries/chaindb/include/chainbase/pinnable_mapped_file.hpp +++ b/libraries/chaindb/include/chainbase/pinnable_mapped_file.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -45,7 +46,24 @@ class chainbase_error_category : public std::error_category { using segment_manager = bip::managed_mapped_file::segment_manager; template -using allocator = bip::allocator; +using segment_allocator_t = bip::allocator; + +using byte_segment_allocator_t = segment_allocator_t; + +using ss_allocator_t = small_size_allocator; + +// An allocator for objects of type T within the segment_manager +// ------------------------------------------------------------- +// - If the allocation size (num_objects * sizeof(T)) is less than 512 bytes, it will be routed +// through the small size allocator which allocates in batch from the `segment_manager`. +// - If the allocation size (num_objects * sizeof(T)) is greater than 512 bytes, the allocator +// will allocate directly from the segment manager. +// - the 512 bytes limit is derived from the template parameters of `small_size_allocator` +// (size_t num_allocators = 64, size_t size_increment = 8) +// - emulates the API of `bip::allocator` +// --------------------------------------------------------------------------------------------- +template +using allocator = object_allocator; class pinnable_mapped_file { public: @@ -66,19 +84,22 @@ class pinnable_mapped_file { segment_manager* get_segment_manager() const { return _segment_manager;} size_t check_memory_and_flush_if_needed(); + static ss_allocator_t* get_small_size_allocator(std::byte* seg_mgr); + template static std::optional> get_allocator(void *object) { if (!_segment_manager_map.empty()) { auto it = _segment_manager_map.upper_bound(object); if(it == _segment_manager_map.begin()) return {}; - auto [seg_start, seg_end] = *(--it); + auto& [seg_start, seg_info] = *(--it); // important: we need to check whether the pointer is really within the segment, as shared objects' // can also be created on the stack (in which case the data is actually allocated on the heap using // std::allocator). This happens for example when `shared_cow_string`s are inserted into a bip::multimap, // and temporary pairs are created on the stack by the bip::multimap code. - if (object < seg_end) - return allocator(reinterpret_cast(seg_start)); + if (object < seg_info.seg_end) { + return std::optional>{allocator(get_small_size_allocator(static_cast(seg_start)))}; + } } return {}; } @@ -114,13 +135,32 @@ class pinnable_mapped_file { static std::vector _instance_tracker; - using segment_manager_map_t = boost::container::flat_map; + struct seg_info_t { void* seg_end; }; + using segment_manager_map_t = boost::container::flat_map; static segment_manager_map_t _segment_manager_map; constexpr static unsigned _db_size_multiple_requirement = 1024*1024; //1MB constexpr static size_t _db_size_copy_increment = 1024*1024*1024; //1GB }; +// There can be at most one `small_size_allocator` per `segment_manager` (hence the `assert` below). +// There is none created if the pinnable_mapped_file is read-only. +// ---------------------------------------------------------------------------------------------------- +template +auto make_small_size_allocator(segment_manager* seg_mgr) { + assert(pinnable_mapped_file::get_small_size_allocator((std::byte*)seg_mgr) == nullptr); + byte_segment_allocator_t byte_allocator(seg_mgr); + return new (seg_mgr->allocate(sizeof(ss_allocator_t))) ss_allocator_t(byte_allocator); +} + +// Create an allocator for a specific object type. +// pointer can be to the segment manager, or any object contained within. +// --------------------------------------------------------------------- +template +auto make_allocator(void* seg_mgr) { + return *pinnable_mapped_file::get_allocator(seg_mgr); +} + std::istream& operator>>(std::istream& in, pinnable_mapped_file::map_mode& runtime); std::ostream& operator<<(std::ostream& osm, pinnable_mapped_file::map_mode m); diff --git a/libraries/chaindb/include/chainbase/shared_cow_string.hpp b/libraries/chaindb/include/chainbase/shared_cow_string.hpp index 4dd96dc53e..32e139f0c3 100644 --- a/libraries/chaindb/include/chainbase/shared_cow_string.hpp +++ b/libraries/chaindb/include/chainbase/shared_cow_string.hpp @@ -26,7 +26,7 @@ namespace chainbase { }; public: - using allocator_type = bip::allocator; + using allocator_type = allocator; using iterator = const char*; using const_iterator = const char*; diff --git a/libraries/chaindb/include/chainbase/shared_cow_vector.hpp b/libraries/chaindb/include/chainbase/shared_cow_vector.hpp index 4af04ff390..7de49e7398 100644 --- a/libraries/chaindb/include/chainbase/shared_cow_vector.hpp +++ b/libraries/chaindb/include/chainbase/shared_cow_vector.hpp @@ -23,7 +23,7 @@ namespace chainbase { }; public: - using allocator_type = bip::allocator; + using allocator_type = allocator; using iterator = const T*; // const because of copy-on-write using const_iterator = const T*; using value_type = T; diff --git a/libraries/chaindb/include/chainbase/small_size_allocator.hpp b/libraries/chaindb/include/chainbase/small_size_allocator.hpp new file mode 100644 index 0000000000..f0978b6277 --- /dev/null +++ b/libraries/chaindb/include/chainbase/small_size_allocator.hpp @@ -0,0 +1,207 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace bip = boost::interprocess; + +namespace chainbase { + +namespace detail { + +// --------------------------------------------------------------------------------------- +// One of the allocators from `small_size_allocator` below +// ------------------------------------------------------- +// +// - allocates buffers of `sz` bytes. +// - allocates in batch from `backing_allocator` (see `allocation_batch_size`) +// - freed buffers are linked into a free list for fast further allocations +// - allocated buffers are never returned to the `backing_allocator` +// - thread-safe +// --------------------------------------------------------------------------------------- +template +class allocator { +public: + using pointer = backing_allocator::pointer; + + allocator(backing_allocator back_alloc, std::size_t sz) + : _sz(sz) + , _back_alloc(back_alloc) {} + + pointer allocate() { + std::lock_guard g(_m); + if (_block_start == _block_end && _freelist == nullptr) { + get_some(); + } + if (_block_start < _block_end) { + pointer result = pointer{_block_start.get()}; + _block_start += _sz; + return result; + } + assert(_freelist != nullptr); + list_item* result = &*_freelist; + _freelist = _freelist->_next; + result->~list_item(); + --_freelist_size; + return pointer{(typename backing_allocator::value_type*)result}; + } + + void deallocate(const pointer& p) { + std::lock_guard g(_m); + _freelist = new (&*p) list_item{_freelist}; + ++_freelist_size; + } + + size_t freelist_memory_usage() const { + std::lock_guard g(_m); + return _freelist_size * _sz + (_block_end - _block_start); + } + + size_t num_blocks_allocated() const { + std::lock_guard g(_m); + return _num_blocks_allocated; + } + +private: + struct list_item { bip::offset_ptr _next; }; + static constexpr size_t max_allocation_batch_size = 512; + + void get_some() { + assert(_sz >= sizeof(list_item)); + assert(_sz % alignof(list_item) == 0); + + _block_start = _back_alloc.allocate(_sz * _allocation_batch_size); + _block_end = _block_start + _sz * _allocation_batch_size; + ++_num_blocks_allocated; + if (_allocation_batch_size < max_allocation_batch_size) + _allocation_batch_size *= 2; + } + + std::size_t _sz; + bip::offset_ptr _freelist; + bip::offset_ptr _block_start; + bip::offset_ptr _block_end; + backing_allocator _back_alloc; + size_t _allocation_batch_size = 32; + size_t _freelist_size = 0; + size_t _num_blocks_allocated = 0; // number of blocks allocated from boost segment allocator + mutable std::mutex _m; +}; + +} // namespace detail + + +// --------------------------------------------------------------------------------------- +// An array of 128 allocators for sizes from 8 to 1024 bytes +// --------------------------------------------------------- +// +// - All pointers used are of type `backing_allocator::pointer` +// - allocate/deallocate specify size in bytes. +// - Any requested size greater than `num_allocators * size_increment` will be routed +// to the backing_allocator +// --------------------------------------------------------------------------------------- +template +requires ((size_increment & (size_increment - 1)) == 0) // power of two +class small_size_allocator { +public: + using pointer = backing_allocator::pointer; + using alloc_ptr = bip::offset_ptr>; + using alloc_array_t = std::array; +private: + backing_allocator _back_alloc; + alloc_array_t _allocators; + + static constexpr size_t max_size = num_allocators * size_increment; + + static constexpr size_t allocator_index(size_t sz_in_bytes) { + assert(sz_in_bytes > 0); + return (sz_in_bytes - 1) / size_increment; + } + + template + auto make_allocators(backing_allocator back_alloc, std::index_sequence) { + return alloc_array_t{new (&*_back_alloc.allocate(sizeof(detail::allocator))) + detail::allocator(back_alloc, (I + 1) * size_increment)...}; + } + +public: + explicit small_size_allocator(backing_allocator back_alloc) + : _back_alloc(std::move(back_alloc)) + , _allocators(make_allocators(back_alloc, std::make_index_sequence{})) {} + + pointer allocate(std::size_t sz_in_bytes) { + if (sz_in_bytes <= max_size) { + return _allocators[allocator_index(sz_in_bytes)]->allocate(); + } + return _back_alloc.allocate(sz_in_bytes); + } + + void deallocate(const pointer& p, std::size_t sz_in_bytes) { + if (sz_in_bytes <= max_size) { + _allocators[allocator_index(sz_in_bytes)]->deallocate(p); + } else + _back_alloc.deallocate(p, sz_in_bytes); + } + + size_t freelist_memory_usage() const { + size_t sz = 0; + for (auto& alloc : _allocators) + sz += alloc->freelist_memory_usage(); + return sz; + } + + size_t num_blocks_allocated() const { + size_t sz = 0; + for (auto& alloc : _allocators) + sz += alloc->num_blocks_allocated(); + return sz; + } +}; + +// --------------------------------------------------------------------------------------- +// Object allocator +// ---------------- +// +// emulates the API of `bip::allocator` +// backing_allocator is normally the `small_size_allocator`, in which case: +// - If the allocation size (num_objects * sizeof(T)) is less than 1024 bytes, it will be routed +// through the small size allocator which allocates in batch from the `segment_manager`. +// - If the allocation size (num_objects * sizeof(T)) is greater than 1024 bytes, the allocator +// will allocate directly from the segment manager. +// - the 1024 bytes limit is derived from the template parameters of `small_size_allocator` +// (size_t num_allocators = 128, size_t size_increment = 8) +// --------------------------------------------------------------------------------------- +template +class object_allocator { +public: + using char_pointer = backing_allocator::pointer; + using pointer = char_pointer::template rebind; + using value_type = T; + + explicit object_allocator(backing_allocator* back_alloc) :_back_alloc(back_alloc) { + } + + pointer allocate(std::size_t num_objects) { + return pointer(static_cast(static_cast(_back_alloc->allocate(num_objects * sizeof(T)).get()))); + } + + void deallocate(const pointer& p, std::size_t num_objects) { + assert(p != nullptr); + return _back_alloc->deallocate(char_pointer(static_cast(static_cast(p.get()))), num_objects * sizeof(T)); + } + + bool operator==(const object_allocator&) const = default; + +private: + bip::offset_ptr _back_alloc; // allocates by size in bytes +}; + +} // namespace chainbase diff --git a/libraries/chaindb/include/chainbase/undo_index.hpp b/libraries/chaindb/include/chainbase/undo_index.hpp index b0b672ac45..81c7f78c43 100644 --- a/libraries/chaindb/include/chainbase/undo_index.hpp +++ b/libraries/chaindb/include/chainbase/undo_index.hpp @@ -349,6 +349,10 @@ namespace chainbase { uint64_t ctime = 0; // _monotonic_revision at the point the undo_state was created }; + void preallocate( std::size_t num ) { + _allocator.preallocate(num); + } + // Exception safety: strong template const value_type& emplace( Constructor&& c ) { diff --git a/libraries/chaindb/src/pinnable_mapped_file.cpp b/libraries/chaindb/src/pinnable_mapped_file.cpp index 88e87324be..2c7b89120f 100644 --- a/libraries/chaindb/src/pinnable_mapped_file.cpp +++ b/libraries/chaindb/src/pinnable_mapped_file.cpp @@ -251,7 +251,20 @@ pinnable_mapped_file::pinnable_mapped_file(const std::filesystem::path& dir, boo } std::byte* start = (std::byte*)_segment_manager; assert(_segment_manager_map.find(start) == _segment_manager_map.end()); - _segment_manager_map[start] = start + _segment_manager->get_size(); + + ss_allocator_t* ss_alloc = get_small_size_allocator(start); // relies on `_segment_manager` being initialized + if (!ss_alloc && _writable) { + // create the unique `small_size_allocator` for this `segment_manager` + db_header* header = reinterpret_cast(start - header_size); + header->small_size_allocator = (char *)make_small_size_allocator(_segment_manager); + } + + _segment_manager_map[start] = seg_info_t{start + _segment_manager->get_size()}; +} + +ss_allocator_t* pinnable_mapped_file::get_small_size_allocator(std::byte* seg_mgr) { + db_header* header = reinterpret_cast(seg_mgr - header_size); + return header->small_size_allocator ? (ss_allocator_t*)&*header->small_size_allocator : nullptr; } void pinnable_mapped_file::setup_copy_on_write_mapping() { @@ -318,8 +331,8 @@ void pinnable_mapped_file::setup_non_file_mapping() { _non_file_mapped_mapping_size = (_non_file_mapped_mapping_size + (r-1u))/r*r; }; - const unsigned _1gb = 1u<<30u; - const unsigned _2mb = 1u<<21u; + [[maybe_unused]] const unsigned _1gb = 1u<<30u; + [[maybe_unused]] const unsigned _2mb = 1u<<21u; #if defined(MAP_HUGETLB) && defined(MAP_HUGE_1GB) _non_file_mapped_mapping = mmap(NULL, _non_file_mapped_mapping_size, PROT_READ|PROT_WRITE, common_map_opts|MAP_HUGETLB|MAP_HUGE_1GB, -1, 0); diff --git a/libraries/chaindb/test/test.cpp b/libraries/chaindb/test/test.cpp index ba9f576e40..e7280a3550 100644 --- a/libraries/chaindb/test/test.cpp +++ b/libraries/chaindb/test/test.cpp @@ -175,9 +175,8 @@ void check_shared_vector_apis(VecOfVec& vec_of_vec, const Alloc& expected_alloc) // check that objects are allocated where we expect (i.e. using the same allocator as `vec_of_vec`) // ------------------------------------------------------------------------------------------------ - BOOST_REQUIRE(v.get_allocator() == expected_alloc); - if constexpr(!std::is_same_v) - BOOST_REQUIRE(v[0].get_allocator() == expected_alloc); + auto alloc = v.get_allocator(); + BOOST_REQUIRE(!alloc || *alloc == expected_alloc); } { @@ -195,6 +194,7 @@ void check_shared_vector_apis(VecOfVec& vec_of_vec, const Alloc& expected_alloc) // check copy constructor. Verify copy-on-write after assign // --------------------------------------------------------- vec_of_vec.clear(); + vec_of_vec.reserve(2); // so the second emplace_back doesn't reallocate vec_of_vec.emplace_back(int_array.cbegin(), int_array.cend()); vec_of_vec.emplace_back(vec_of_vec[0]); auto& v0 = vec_of_vec[0]; @@ -212,6 +212,7 @@ void check_shared_vector_apis(VecOfVec& vec_of_vec, const Alloc& expected_alloc) // check move constructor. // ----------------------- vec_of_vec.clear(); + vec_of_vec.reserve(2); // so the second emplace_back doesn't reallocate vec_of_vec.emplace_back(int_array.cbegin(), int_array.cend()); auto& v0 = vec_of_vec[0]; vec_of_vec.emplace_back(std::move(v0)); @@ -453,32 +454,34 @@ BOOST_AUTO_TEST_CASE(shared_vector_apis_segment_alloc) { const auto& temp = temp_dir.path(); pinnable_mapped_file pmf(temp, true, 1024 * 1024, false, pinnable_mapped_file::map_mode::mapped); - std::optional> expected_alloc = chainbase::allocator(pmf.get_segment_manager()); + auto seg_mgr = pmf.get_segment_manager(); - size_t free_memory = pmf.get_segment_manager()->get_free_memory(); + size_t free_memory = seg_mgr->get_free_memory(); { // do the test with `shared_vector` (trivial destructor) // ---------------------------------------------------------- - using sv = shared_vector; - chainbase::allocator sv_alloc(pmf.get_segment_manager()); - sv v; + using sv_t = shared_vector; + auto sv_alloc = chainbase::make_allocator(seg_mgr); - bip::vector> vec_of_vec(sv_alloc); + using vec_of_vec_t = bip::vector; + vec_of_vec_t vec_of_vec(sv_alloc); - check_shared_vector_apis(vec_of_vec, expected_alloc); + check_shared_vector_apis(vec_of_vec, chainbase::make_allocator(seg_mgr)); } { // do the test with `shared_vector` (non-trivial destructor) // -------------------------------------------------------------------- - using sv = shared_vector; - chainbase::allocator sv_alloc(pmf.get_segment_manager()); - sv v; + using sv_t = shared_vector; + auto sv_alloc = chainbase::make_allocator(seg_mgr); + + using vec_of_vec_t = bip::vector; + vec_of_vec_t vec_of_vec(sv_alloc); - bip::vector> vec_of_vec(sv_alloc); + sv_t v; - check_shared_vector_apis(vec_of_vec, expected_alloc); + check_shared_vector_apis(vec_of_vec, chainbase::make_allocator(seg_mgr)); // clear both vectors. If our implementation of `shared_cow_vector` is correct, we should have an exact // match of the number of constructed and destroyed `my_string` objects, and therefore after clearing the vectors @@ -491,7 +494,12 @@ BOOST_AUTO_TEST_CASE(shared_vector_apis_segment_alloc) { // make sure we didn't leak memory // ------------------------------- - BOOST_REQUIRE_EQUAL(free_memory, pmf.get_segment_manager()->get_free_memory()); + auto ss_alloc = pinnable_mapped_file::get_small_size_allocator((std::byte*)pmf.get_segment_manager()); + auto num_blocks_allocated = ss_alloc->num_blocks_allocated(); + auto lost = free_memory - (seg_mgr->get_free_memory() + ss_alloc->freelist_memory_usage()); + + // for every block allocated from the shared memory segment, we have an overhead of 8 or 16 bytes + BOOST_REQUIRE(lost == num_blocks_allocated * 8 || lost == num_blocks_allocated * 16); } // -----------------------------------------------------------------------------