From 81c6338dfe93a86b49338ef78e95d51ee591bd08 Mon Sep 17 00:00:00 2001 From: Nathan Hourt Date: Fri, 12 Jun 2015 11:15:11 -0400 Subject: [PATCH 1/7] Begin test for cashback rewards --- .../include/graphene/chain/operations.hpp | 2 +- libraries/chain/operations.cpp | 3 +- tests/common/database_fixture.hpp | 6 +- tests/tests/fee_tests.cpp | 83 +++++++++++++++++++ 4 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 tests/tests/fee_tests.cpp diff --git a/libraries/chain/include/graphene/chain/operations.hpp b/libraries/chain/include/graphene/chain/operations.hpp index b670be1d..7be5368a 100644 --- a/libraries/chain/include/graphene/chain/operations.hpp +++ b/libraries/chain/include/graphene/chain/operations.hpp @@ -132,7 +132,7 @@ namespace graphene { namespace chain { account_id_type referrer; /// Of the fee split between registrar and referrer, this percentage goes to the referrer. The rest goes to the /// registrar. - uint8_t referrer_percent = 0; + uint16_t referrer_percent = 0; string name; authority owner; diff --git a/libraries/chain/operations.cpp b/libraries/chain/operations.cpp index c08027e2..5e88efed 100644 --- a/libraries/chain/operations.cpp +++ b/libraries/chain/operations.cpp @@ -201,8 +201,7 @@ void account_create_operation::validate()const { FC_ASSERT( fee.amount >= 0 ); FC_ASSERT( is_valid_name( name ) ); - FC_ASSERT( referrer_percent >= 0 ); - FC_ASSERT( referrer_percent <= 100 ); + FC_ASSERT( referrer_percent <= GRAPHENE_100_PERCENT ); FC_ASSERT( !owner.auths.empty() ); auto pos = name.find( '/' ); if( pos != string::npos ) diff --git a/tests/common/database_fixture.hpp b/tests/common/database_fixture.hpp index 66b2ceeb..a841aaf1 100644 --- a/tests/common/database_fixture.hpp +++ b/tests/common/database_fixture.hpp @@ -67,9 +67,11 @@ using namespace graphene::db; #define PUSH_TX( tx, skip_flags ) \ _push_transaction( tx, skip_flags, __FILE__, __LINE__ ) -#define ACTOR(name) \ +#define PREP_ACTOR(name) \ fc::ecc::private_key name ## _private_key = generate_private_key(BOOST_PP_STRINGIZE(name)); \ - key_id_type name ## _key_id = register_key(name ## _private_key.get_public_key()).get_id(); \ + key_id_type name ## _key_id = register_key(name ## _private_key.get_public_key()).get_id(); +#define ACTOR(name) \ + PREP_ACTOR(name) \ account_id_type name ## _id = create_account(BOOST_PP_STRINGIZE(name), name ## _key_id).id; #define GET_ACTOR(name) \ fc::ecc::private_key name ## _private_key = generate_private_key(BOOST_PP_STRINGIZE(name)); \ diff --git a/tests/tests/fee_tests.cpp b/tests/tests/fee_tests.cpp new file mode 100644 index 00000000..03816ecb --- /dev/null +++ b/tests/tests/fee_tests.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2015, Cryptonomex, Inc. + * All rights reserved. + * + * This source code is provided for evaluation in private test networks only, until September 8, 2015. After this date, this license expires and + * the code may not be used, modified or distributed for any purpose. Redistribution and use in source and binary forms, with or without modification, + * are permitted until September 8, 2015, provided that the following conditions are met: + * + * 1. The code and/or derivative works are used only for private test networks consisting of no more than 10 P2P nodes. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "../common/database_fixture.hpp" + +using namespace graphene::chain; + +BOOST_FIXTURE_TEST_SUITE( fee_tests, database_fixture ) + +BOOST_AUTO_TEST_CASE( cashback_test ) +{ try { + /* Account Structure used in this test * + * * + * /-----------------\ /-------------------\ * + * | life (Lifetime) | | reggie (Lifetime) | * + * \-----------------/ \-------------------/ * + * | Refers & | Refers | Registers | Registers * + * v Registers v v | * + * /----------------\ /----------------\ | * + * | ann (Annual) | | dumy (basic) | | * + * \----------------/ \----------------/ |-------------. * + * | Refers & L--------------------------------. | | * + * v Registers Refers v v | * + * /----------------\ /----------------\ | * + * | scud (basic) | | stud (basic) | | * + * \----------------/ | (Upgrades to | | * + * | Lifetime) | v * + * \----------------/ /--------------\ * + * L------->| pleb (Basic) | * + * \--------------/ * + * * + */ + ACTOR(life); + ACTOR(reggie); + PREP_ACTOR(ann); + PREP_ACTOR(scud); + PREP_ACTOR(dumy); + PREP_ACTOR(stud); + PREP_ACTOR(pleb); + transfer(account_id_type(), life_id, asset(100000000)); + transfer(account_id_type(), reggie_id, asset(100000000)); + upgrade_to_lifetime_member(life_id); + upgrade_to_lifetime_member(reggie_id); + enable_fees(10000); + const auto& fees = db.get_global_properties().parameters.current_fees; + +#define CustomRegisterActor(actor_name, registrar_name, referrer_name, referrer_rate) \ + { \ + account_create_operation op; \ + op.registrar = registrar_name ## _id; \ + op.referrer = referrer_name ## _id; \ + op.referrer_percent = referrer_rate*GRAPHENE_1_PERCENT; \ + op.name = BOOST_PP_STRINGIZE(actor_name); \ + op.memo_key = actor_name ## _key_id; \ + op.active = authority(1, actor_name ## _key_id, 1); \ + op.owner = op.active; \ + op.fee = op.calculate_fee(fees); \ + trx.operations = {op}; \ + trx.sign(registrar_name ## _key_id, registrar_name ## _private_key); \ + db.push_transaction(trx); \ + } + + CustomRegisterActor(ann, life, life, 75); +} FC_LOG_AND_RETHROW() } + +BOOST_AUTO_TEST_SUITE_END() From ce4846e81b4848f680f4924bb488ff247355dd30 Mon Sep 17 00:00:00 2001 From: Nathan Hourt Date: Mon, 15 Jun 2015 17:28:20 -0400 Subject: [PATCH 2/7] Progress #31: Add some testing --- libraries/chain/account_evaluator.cpp | 2 + libraries/chain/account_object.cpp | 75 +++++++++++++++++++ libraries/chain/db_maint.cpp | 71 +----------------- .../include/graphene/chain/account_object.hpp | 13 +++- .../graphene/chain/vesting_balance_object.hpp | 3 +- tests/common/database_fixture.cpp | 31 +++++++- tests/common/database_fixture.hpp | 6 +- tests/tests/fee_tests.cpp | 36 +++++++-- 8 files changed, 152 insertions(+), 85 deletions(-) diff --git a/libraries/chain/account_evaluator.cpp b/libraries/chain/account_evaluator.cpp index 0e9aed33..c8ef51f8 100644 --- a/libraries/chain/account_evaluator.cpp +++ b/libraries/chain/account_evaluator.cpp @@ -224,6 +224,7 @@ void_result account_upgrade_evaluator::do_apply(const account_upgrade_evaluator: if( o.upgrade_to_lifetime_member ) { // Upgrade to lifetime member. I don't care what the account was before. + a.statistics(d).process_fees(a, d); a.membership_expiration_date = time_point_sec::maximum(); a.referrer = a.registrar = a.lifetime_referrer = a.get_id(); a.lifetime_referrer_fee_percentage = GRAPHENE_100_PERCENT - a.network_fee_percentage; @@ -234,6 +235,7 @@ void_result account_upgrade_evaluator::do_apply(const account_upgrade_evaluator: a.membership_expiration_date += fc::days(365); } else { // Upgrade from basic account. + a.statistics(d).process_fees(a, d); assert(a.is_basic_account(d.head_block_time())); a.membership_expiration_date = d.head_block_time() + fc::days(365); } diff --git a/libraries/chain/account_object.cpp b/libraries/chain/account_object.cpp index 59ba30aa..512974e6 100644 --- a/libraries/chain/account_object.cpp +++ b/libraries/chain/account_object.cpp @@ -17,10 +17,24 @@ */ #include #include +#include #include namespace graphene { namespace chain { +share_type cut_fee(share_type a, uint16_t p) +{ + if( a == 0 || p == 0 ) + return 0; + if( p == GRAPHENE_100_PERCENT ) + return a; + + fc::uint128 r(a.value); + r *= p; + r /= GRAPHENE_100_PERCENT; + return r.to_uint64(); +} + bool account_object::is_authorized_asset(const asset_object& asset_obj) const { for( const auto id : blacklisting_accounts ) if( asset_obj.options.blacklist_authorities.find(id) != asset_obj.options.blacklist_authorities.end() ) return false; @@ -56,4 +70,65 @@ uint16_t account_statistics_object::calculate_bulk_discount_percent(const chain_ return bulk_discount_percent; } +void account_statistics_object::process_fees(const account_object& a, database& d) const +{ + if( pending_fees > 0 || pending_vested_fees > 0 ) + { + const auto& props = d.get_global_properties(); + + auto pay_out_fees = [&](const account_object& account, share_type core_fee_total, bool require_vesting) + { + share_type network_cut = cut_fee(core_fee_total, account.network_fee_percentage); + assert( network_cut <= core_fee_total ); + share_type burned = cut_fee(network_cut, props.parameters.burn_percent_of_fee); + share_type accumulated = network_cut - burned; + assert( accumulated + burned == network_cut ); + share_type lifetime_cut = cut_fee(core_fee_total, account.lifetime_referrer_fee_percentage); + share_type referral = core_fee_total - network_cut - lifetime_cut; + + d.modify(dynamic_asset_data_id_type()(d), [network_cut](asset_dynamic_data_object& d) { + d.accumulated_fees += network_cut; + }); + + // Potential optimization: Skip some of this math and object lookups by special casing on the account type. + // For example, if the account is a lifetime member, we can skip all this and just deposit the referral to + // it directly. + share_type referrer_cut = cut_fee(referral, account.referrer_rewards_percentage); + share_type registrar_cut = referral - referrer_cut; + + d.deposit_cashback(d.get(account.lifetime_referrer), lifetime_cut, require_vesting); + d.deposit_cashback(d.get(account.referrer), referrer_cut, require_vesting); + d.deposit_cashback(d.get(account.registrar), registrar_cut, require_vesting); + + assert( referrer_cut + registrar_cut + accumulated + burned + lifetime_cut == core_fee_total ); + }; + + share_type vesting_fee_subtotal(pending_fees); + share_type vested_fee_subtotal(pending_vested_fees); + share_type vesting_cashback, vested_cashback; + + if( lifetime_fees_paid > props.parameters.bulk_discount_threshold_min && + a.is_member(d.head_block_time()) ) + { + auto bulk_discount_rate = calculate_bulk_discount_percent(props.parameters); + vesting_cashback = cut_fee(vesting_fee_subtotal, bulk_discount_rate); + vesting_fee_subtotal -= vesting_cashback; + + vested_cashback = cut_fee(vested_fee_subtotal, bulk_discount_rate); + vested_fee_subtotal -= vested_cashback; + } + + pay_out_fees(a, vesting_fee_subtotal, true); + d.deposit_cashback(a, vesting_cashback, true); + pay_out_fees(a, vested_fee_subtotal, false); + d.deposit_cashback(a, vested_cashback, false); + + d.modify(*this, [vested_fee_subtotal, vesting_fee_subtotal](account_statistics_object& s) { + s.lifetime_fees_paid += vested_fee_subtotal + vesting_fee_subtotal; + s.pending_fees = 0; + s.pending_vested_fees = 0; + }); + } +} + } } // graphene::chain diff --git a/libraries/chain/db_maint.cpp b/libraries/chain/db_maint.cpp index 0db0b6f6..23f16537 100644 --- a/libraries/chain/db_maint.cpp +++ b/libraries/chain/db_maint.cpp @@ -357,77 +357,8 @@ void database::perform_chain_maintenance(const signed_block& next_block, const g process_fees_helper(database& d, const global_property_object& gpo) : d(d), props(gpo) {} - share_type cut_fee(share_type a, uint16_t p)const - { - if( a == 0 || p == 0 ) - return 0; - if( p == GRAPHENE_100_PERCENT ) - return a; - - fc::uint128 r(a.value); - r *= p; - r /= GRAPHENE_100_PERCENT; - return r.to_uint64(); - } - - void pay_out_fees(const account_object& account, share_type core_fee_total, bool require_vesting) - { - share_type network_cut = cut_fee(core_fee_total, account.network_fee_percentage); - assert( network_cut <= core_fee_total ); - share_type burned = cut_fee(network_cut, props.parameters.burn_percent_of_fee); - share_type accumulated = network_cut - burned; - assert( accumulated + burned == network_cut ); - share_type lifetime_cut = cut_fee(core_fee_total, account.lifetime_referrer_fee_percentage); - share_type referral = core_fee_total - network_cut - lifetime_cut; - - d.modify(dynamic_asset_data_id_type()(d), [network_cut](asset_dynamic_data_object& d) { - d.accumulated_fees += network_cut; - }); - - // Potential optimization: Skip some of this math and object lookups by special casing on the account type. - // For example, if the account is a lifetime member, we can skip all this and just deposit the referral to - // it directly. - share_type referrer_cut = cut_fee(referral, account.referrer_rewards_percentage); - share_type registrar_cut = referral - referrer_cut; - - d.deposit_cashback(d.get(account.lifetime_referrer), lifetime_cut, require_vesting); - d.deposit_cashback(d.get(account.referrer), referrer_cut, require_vesting); - d.deposit_cashback(d.get(account.registrar), registrar_cut, require_vesting); - - assert( referrer_cut + registrar_cut + accumulated + burned + lifetime_cut == core_fee_total ); - } - void operator()(const account_object& a) { - const account_statistics_object& stats = a.statistics(d); - - if( stats.pending_fees > 0 ) - { - share_type vesting_fee_subtotal(stats.pending_fees); - share_type vested_fee_subtotal(stats.pending_vested_fees); - share_type vesting_cashback, vested_cashback; - - if( stats.lifetime_fees_paid > props.parameters.bulk_discount_threshold_min && - a.is_member(d.head_block_time()) ) - { - auto bulk_discount_rate = stats.calculate_bulk_discount_percent(props.parameters); - vesting_cashback = cut_fee(vesting_fee_subtotal, bulk_discount_rate); - vesting_fee_subtotal -= vesting_cashback; - - vested_cashback = cut_fee(vested_fee_subtotal, bulk_discount_rate); - vested_fee_subtotal -= vested_cashback; - } - - pay_out_fees(a, vesting_fee_subtotal, true); - d.deposit_cashback(a, vesting_cashback, true); - pay_out_fees(a, vested_fee_subtotal, false); - d.deposit_cashback(a, vested_cashback, false); - - d.modify(stats, [vested_fee_subtotal, vesting_fee_subtotal](account_statistics_object& s) { - s.lifetime_fees_paid += vested_fee_subtotal + vesting_fee_subtotal; - s.pending_fees = 0; - s.pending_vested_fees = 0; - }); - } + a.statistics(d).process_fees(a, d); } } fee_helper(*this, gpo); diff --git a/libraries/chain/include/graphene/chain/account_object.hpp b/libraries/chain/include/graphene/chain/account_object.hpp index c41a15fc..0a3497d6 100644 --- a/libraries/chain/include/graphene/chain/account_object.hpp +++ b/libraries/chain/include/graphene/chain/account_object.hpp @@ -22,6 +22,7 @@ #include namespace graphene { namespace chain { +class database; /** * @class account_statistics_object @@ -64,8 +65,8 @@ namespace graphene { namespace chain { * them yet (registrar, referrer, lifetime referrer, network, etc). This is used as an optimization to avoid * doing massive amounts of uint128 arithmetic on each and every operation. * - *These fees will be paid out as vesting cash-back, and this counter will reset during the maintenance - *interval. + * These fees will be paid out as vesting cash-back, and this counter will reset during the maintenance + * interval. */ share_type pending_fees; /** @@ -76,6 +77,8 @@ namespace graphene { namespace chain { /// @brief Calculate the percentage discount this user receives on his fees uint16_t calculate_bulk_discount_percent(const chain_parameters& params)const; + /// @brief Split up and pay out @ref pending_fees and @ref pending_vested_fees + void process_fees(const account_object& a, database& d) const; }; /** @@ -189,6 +192,12 @@ namespace graphene { namespace chain { * Vesting balance which receives cashback_reward deposits. */ optional cashback_vb; + template + const vesting_balance_object& cashback_balance(const DB& db)const + { + FC_ASSERT(cashback_vb); + return db.get(*cashback_vb); + } /// @return true if this is a lifetime member account; false otherwise. bool is_lifetime_member()const diff --git a/libraries/chain/include/graphene/chain/vesting_balance_object.hpp b/libraries/chain/include/graphene/chain/vesting_balance_object.hpp index c0e32347..3d6ff6fb 100644 --- a/libraries/chain/include/graphene/chain/vesting_balance_object.hpp +++ b/libraries/chain/include/graphene/chain/vesting_balance_object.hpp @@ -98,8 +98,7 @@ namespace graphene { namespace chain { > vesting_policy; /** - * Timelocked balance object is a balance that is locked by the - * blockchain for a period of time. + * Vesting balance object is a balance that is locked by the blockchain for a period of time. */ class vesting_balance_object : public abstract_object { diff --git a/tests/common/database_fixture.cpp b/tests/common/database_fixture.cpp index 27d57445..61a5c8cf 100644 --- a/tests/common/database_fixture.cpp +++ b/tests/common/database_fixture.cpp @@ -270,11 +270,17 @@ void database_fixture::generate_blocks( uint32_t block_count ) generate_block(); } -void database_fixture::generate_blocks( fc::time_point_sec timestamp ) +void database_fixture::generate_blocks(fc::time_point_sec timestamp, bool miss_intermediate_blocks) { + if( miss_intermediate_blocks ) + { + auto slots_to_miss = db.get_slot_at_time(timestamp) - 1; + assert(slots_to_miss > 0); + generate_block(~0, generate_private_key("genesis"), slots_to_miss); + return; + } while( db.head_block_time() < timestamp ) generate_block(); - return; } account_create_operation database_fixture::make_account( @@ -683,14 +689,33 @@ void database_fixture::upgrade_to_lifetime_member( const account_object& account account_upgrade_operation op; op.account_to_upgrade = account.get_id(); op.upgrade_to_lifetime_member = true; + op.fee = op.calculate_fee(db.get_global_properties().parameters.current_fees); trx.operations = {op}; - db.push_transaction( trx, ~0 ); + db.push_transaction(trx, ~0); FC_ASSERT( op.account_to_upgrade(db).is_lifetime_member() ); trx.clear(); } FC_CAPTURE_AND_RETHROW((account)) } +void database_fixture::upgrade_to_annual_member(account_id_type account) +{ + upgrade_to_annual_member(account(db)); +} + +void database_fixture::upgrade_to_annual_member(const account_object& account) +{ + try { + account_upgrade_operation op; + op.account_to_upgrade = account.get_id(); + op.fee = op.calculate_fee(db.get_global_properties().parameters.current_fees); + trx.operations = {op}; + db.push_transaction(trx, ~0); + FC_ASSERT( op.account_to_upgrade(db).is_member(db.head_block_time()) ); + trx.clear(); + } FC_CAPTURE_AND_RETHROW((account)) +} + void database_fixture::print_market( const string& syma, const string& symb )const { const auto& limit_idx = db.get_index_type(); diff --git a/tests/common/database_fixture.hpp b/tests/common/database_fixture.hpp index a841aaf1..7010a29f 100644 --- a/tests/common/database_fixture.hpp +++ b/tests/common/database_fixture.hpp @@ -119,13 +119,13 @@ struct database_fixture { * @brief Generates block_count blocks * @param block_count number of blocks to generate */ - void generate_blocks( uint32_t block_count ); + void generate_blocks(uint32_t block_count); /** * @brief Generates blocks until the head block time matches or exceeds timestamp * @param timestamp target time to generate blocks until */ - void generate_blocks( fc::time_point_sec timestamp ); + void generate_blocks(fc::time_point_sec timestamp, bool miss_intermediate_blocks = false); account_create_operation make_account( const std::string& name = "nathan", @@ -206,6 +206,8 @@ struct database_fixture { void enable_fees( share_type fee = GRAPHENE_BLOCKCHAIN_PRECISION ); void upgrade_to_lifetime_member( account_id_type account ); void upgrade_to_lifetime_member( const account_object& account ); + void upgrade_to_annual_member( account_id_type account ); + void upgrade_to_annual_member( const account_object& account ); void print_market( const string& syma, const string& symb )const; string pretty( const asset& a )const; void print_short_order( const short_order_object& cur )const; diff --git a/tests/tests/fee_tests.cpp b/tests/tests/fee_tests.cpp index 03816ecb..e72ca4d0 100644 --- a/tests/tests/fee_tests.cpp +++ b/tests/tests/fee_tests.cpp @@ -16,6 +16,8 @@ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include + #include #include "../common/database_fixture.hpp" @@ -36,15 +38,15 @@ BOOST_AUTO_TEST_CASE( cashback_test ) * /----------------\ /----------------\ | * * | ann (Annual) | | dumy (basic) | | * * \----------------/ \----------------/ |-------------. * - * | Refers & L--------------------------------. | | * - * v Registers Refers v v | * + * | Refers L--------------------------------. | | * + * v Refers v v | * * /----------------\ /----------------\ | * - * | scud (basic) | | stud (basic) | | * - * \----------------/ | (Upgrades to | | * + * | scud (basic) |<------------------------| stud (basic) | | * + * \----------------/ Registers | (Upgrades to | | * * | Lifetime) | v * * \----------------/ /--------------\ * * L------->| pleb (Basic) | * - * \--------------/ * + * Refers \--------------/ * * * */ ACTOR(life); @@ -62,6 +64,7 @@ BOOST_AUTO_TEST_CASE( cashback_test ) const auto& fees = db.get_global_properties().parameters.current_fees; #define CustomRegisterActor(actor_name, registrar_name, referrer_name, referrer_rate) \ + account_id_type actor_name ## _id; \ { \ account_create_operation op; \ op.registrar = registrar_name ## _id; \ @@ -74,10 +77,31 @@ BOOST_AUTO_TEST_CASE( cashback_test ) op.fee = op.calculate_fee(fees); \ trx.operations = {op}; \ trx.sign(registrar_name ## _key_id, registrar_name ## _private_key); \ - db.push_transaction(trx); \ + actor_name ## _id = db.push_transaction(trx).operation_results.front().get(); \ + trx.clear(); \ } CustomRegisterActor(ann, life, life, 75); + + transfer(life_id, ann_id, asset(1000000)); + upgrade_to_annual_member(ann_id); + + CustomRegisterActor(dumy, reggie, life, 75); + CustomRegisterActor(stud, reggie, ann, 80); + + transfer(life_id, stud_id, asset(1000000)); + upgrade_to_lifetime_member(stud_id); + + CustomRegisterActor(pleb, reggie, stud, 95); + CustomRegisterActor(scud, stud, ann, 80); + + generate_block(); + generate_blocks(db.get_dynamic_global_properties().next_maintenance_time, true); + + BOOST_CHECK_EQUAL(life_id(db).cashback_balance(db).balance.amount.value, 78000); + BOOST_CHECK_EQUAL(reggie_id(db).cashback_balance(db).balance.amount.value, 34000); + BOOST_CHECK_EQUAL(ann_id(db).cashback_balance(db).balance.amount.value, 40000); + BOOST_CHECK_EQUAL(stud_id(db).cashback_balance(db).balance.amount.value, 8000); } FC_LOG_AND_RETHROW() } BOOST_AUTO_TEST_SUITE_END() From e8bbdc08f65dbdaae4d294ba1247ce575190c509 Mon Sep 17 00:00:00 2001 From: Vikram Rajkumar Date: Mon, 15 Jun 2015 16:41:52 -0400 Subject: [PATCH 3/7] Update docs submodule --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 1b53a8ec..71ed2984 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1b53a8eca77783d073ce7cc95991447c3f34b927 +Subproject commit 71ed2984b71d57cab13cdf12074cff150edc1d3d From 377432453fd41f4ee12a7ad7a9b66c64c2d3f600 Mon Sep 17 00:00:00 2001 From: Nathan Hourt Date: Tue, 16 Jun 2015 14:42:02 -0400 Subject: [PATCH 4/7] Lay to rest #16 It's finally done. --- libraries/chain/account_evaluator.cpp | 82 ++++++++----------- libraries/chain/account_object.cpp | 15 ++++ libraries/chain/db_init.cpp | 6 +- libraries/chain/db_maint.cpp | 15 ++-- libraries/chain/evaluator.cpp | 10 ++- .../include/graphene/chain/account_object.hpp | 42 ++++++---- .../chain/include/graphene/chain/config.hpp | 1 - .../include/graphene/chain/evaluator.hpp | 1 + .../include/graphene/chain/operations.hpp | 33 ++++---- libraries/chain/operations.cpp | 11 ++- .../account_history_plugin.cpp | 2 +- libraries/utilities/git_revision.cpp.in | 4 +- libraries/wallet/wallet.cpp | 22 ++--- tests/common/database_fixture.cpp | 16 ++-- tests/tests/authority_tests.cpp | 27 +++--- tests/tests/block_tests.cpp | 4 +- tests/tests/fee_tests.cpp | 2 +- tests/tests/operation_tests.cpp | 18 ++-- tests/tests/operation_tests2.cpp | 22 ++--- 19 files changed, 183 insertions(+), 150 deletions(-) diff --git a/libraries/chain/account_evaluator.cpp b/libraries/chain/account_evaluator.cpp index c8ef51f8..9d059393 100644 --- a/libraries/chain/account_evaluator.cpp +++ b/libraries/chain/account_evaluator.cpp @@ -25,18 +25,19 @@ namespace graphene { namespace chain { void_result account_create_evaluator::do_evaluate( const account_create_operation& op ) { try { database& d = db(); - FC_ASSERT( d.find_object(op.voting_account) ); - FC_ASSERT( is_relative(op.memo_key) || d.find_object(op.memo_key) ); + FC_ASSERT( d.find_object(op.options.voting_account) ); + FC_ASSERT( is_relative(op.options.memo_key) || d.find_object(op.options.memo_key) ); FC_ASSERT( fee_paying_account->is_lifetime_member() ); FC_ASSERT( op.referrer(d).is_member(d.head_block_time()) ); const auto& global_props = d.get_global_properties(); - uint32_t max_vote_id = global_props.next_available_vote_id; const auto& chain_params = global_props.parameters; - FC_ASSERT( op.num_witness <= chain_params.maximum_witness_count ); - FC_ASSERT( op.num_committee <= chain_params.maximum_committee_count ); + FC_ASSERT( op.owner.auths.size() <= chain_params.maximum_authority_membership ); FC_ASSERT( op.active.auths.size() <= chain_params.maximum_authority_membership ); + check_relative_ids(op.owner); + check_relative_ids(op.active); + FC_ASSERT( d.find(key_id_type(get_relative_id(op.options.memo_key))) ); for( auto id : op.owner.auths ) { FC_ASSERT( is_relative(id.first) || d.find_object(id.first) ); @@ -45,18 +46,22 @@ void_result account_create_evaluator::do_evaluate( const account_create_operatio { FC_ASSERT( is_relative(id.first) || d.find_object(id.first) ); } + + uint32_t max_vote_id = global_props.next_available_vote_id; + FC_ASSERT( op.options.num_witness <= chain_params.maximum_witness_count ); + FC_ASSERT( op.options.num_committee <= chain_params.maximum_committee_count ); safe counts[vote_id_type::VOTE_TYPE_COUNT]; - for( auto id : op.vote ) + for( auto id : op.options.votes ) { FC_ASSERT( id < max_vote_id ); counts[id.type()]++; } - FC_ASSERT(counts[vote_id_type::witness] <= op.num_witness, + FC_ASSERT(counts[vote_id_type::witness] <= op.options.num_witness, "", - ("count", counts[vote_id_type::witness])("num", op.num_witness)); - FC_ASSERT(counts[vote_id_type::committee] <= op.num_committee, + ("count", counts[vote_id_type::witness])("num", op.options.num_witness)); + FC_ASSERT(counts[vote_id_type::committee] <= op.options.num_committee, "", - ("count", counts[vote_id_type::committee])("num", op.num_committee)); + ("count", counts[vote_id_type::committee])("num", op.options.num_committee)); auto& acnt_indx = d.get_index_type(); if( op.name.size() ) @@ -65,17 +70,6 @@ void_result account_create_evaluator::do_evaluate( const account_create_operatio FC_ASSERT( current_account_itr == acnt_indx.indices().get().end() ); } - // TODO: this check can be removed after GRAPHENE_LEGACY_NAME_IMPORT_PERIOD - // legacy account check - if( d.get_dynamic_global_properties().head_block_number < GRAPHENE_LEGACY_NAME_IMPORT_PERIOD ) - { - auto legacy_account_itr = acnt_indx.indices().get().find( "bts-"+op.name ); - if( legacy_account_itr != acnt_indx.indices().get().end() ) - { - FC_ASSERT( fee_paying_account->id == legacy_account_itr->id ); - } - } - // verify child account authority auto pos = op.name.find( '/' ); if( pos != string::npos ) @@ -93,9 +87,6 @@ void_result account_create_evaluator::do_evaluate( const account_create_operatio object_id_type account_create_evaluator::do_apply( const account_create_operation& o ) { try { - auto owner = resolve_relative_ids( o.owner ); - auto active = resolve_relative_ids( o.active ); - const auto& stats_obj = db().create( [&]( account_statistics_object& ){ }); @@ -110,14 +101,11 @@ object_id_type account_create_evaluator::do_apply( const account_create_operatio obj.referrer_rewards_percentage = o.referrer_percent; obj.name = o.name; - obj.owner = owner; - obj.active = active; + obj.owner = resolve_relative_ids(o.owner); + obj.active = resolve_relative_ids(o.active); obj.statistics = stats_obj.id; - obj.memo_key = get_relative_id(o.memo_key); - obj.voting_account = o.voting_account; - obj.votes = o.vote; - obj.num_witness = o.num_witness; - obj.num_committee = o.num_committee; + obj.options = o.options; + obj.options.memo_key = get_relative_id(obj.options.memo_key); }); return new_acnt_object.id; @@ -128,14 +116,12 @@ void_result account_update_evaluator::do_evaluate( const account_update_operatio { database& d = db(); - FC_ASSERT( !o.memo_key || is_relative(*o.memo_key) || db().find_object(*o.memo_key) ); - const auto& chain_params = db().get_global_properties().parameters; - FC_ASSERT( o.num_witness <= chain_params.maximum_witness_count ); - FC_ASSERT( o.num_committee <= chain_params.maximum_committee_count ); + if( o.owner ) { FC_ASSERT( o.owner->auths.size() <= chain_params.maximum_authority_membership ); + check_relative_ids(*o.owner); for( auto id : o.owner->auths ) { FC_ASSERT( is_relative(id.first) || db().find(id.first) ); @@ -144,6 +130,7 @@ void_result account_update_evaluator::do_evaluate( const account_update_operatio if( o.active ) { FC_ASSERT( o.active->auths.size() <= chain_params.maximum_authority_membership ); + check_relative_ids(*o.active); for( auto id : o.active->auths ) { FC_ASSERT( is_relative(id.first) || db().find(id.first) ); @@ -152,10 +139,13 @@ void_result account_update_evaluator::do_evaluate( const account_update_operatio acnt = &o.account(d); - if( o.vote ) + if( o.new_options ) { + FC_ASSERT( d.find(key_id_type(get_relative_id(o.new_options->memo_key))) ); + FC_ASSERT( o.new_options->num_witness <= chain_params.maximum_witness_count ); + FC_ASSERT( o.new_options->num_committee <= chain_params.maximum_committee_count ); uint32_t max_vote_id = d.get_global_properties().next_available_vote_id; - for( auto id : *o.vote ) + for( auto id : o.new_options->votes ) { FC_ASSERT( id < max_vote_id ); } @@ -165,15 +155,15 @@ void_result account_update_evaluator::do_evaluate( const account_update_operatio } void_result account_update_evaluator::do_apply( const account_update_operation& o ) { - db().modify( *acnt, [&]( account_object& a ){ - if( o.owner ) a.owner = *o.owner; - if( o.active ) a.active = *o.active; - if( o.voting_account ) a.voting_account = *o.voting_account; - if( o.memo_key ) a.memo_key = *o.memo_key; - if( o.vote ) a.votes = *o.vote; - a.num_witness = o.num_witness; - a.num_committee = o.num_committee; - }); + db().modify( *acnt, [&](account_object& a){ + if( o.owner ) a.owner = resolve_relative_ids(*o.owner); + if( o.active ) a.active = resolve_relative_ids(*o.active); + if( o.new_options ) + { + a.options = *o.new_options; + a.options.memo_key = get_relative_id(a.options.memo_key); + } + }); return void_result(); } diff --git a/libraries/chain/account_object.cpp b/libraries/chain/account_object.cpp index 512974e6..37f07785 100644 --- a/libraries/chain/account_object.cpp +++ b/libraries/chain/account_object.cpp @@ -131,4 +131,19 @@ void account_statistics_object::process_fees(const account_object& a, database& } } +void account_object::options_type::validate() const +{ + auto needed_witnesses = num_witness; + auto needed_committee = num_committee; + + for( vote_id_type id : votes ) + if( id.type() == vote_id_type::witness && needed_witnesses ) + --needed_witnesses; + else if ( id.type() == vote_id_type::committee && needed_committee ) + --needed_committee; + + FC_ASSERT( needed_witnesses == 0 && needed_committee == 0, + "May not specify fewer witnesses or committee members than the number voted for."); +} + } } // graphene::chain diff --git a/libraries/chain/db_init.cpp b/libraries/chain/db_init.cpp index 57c3cd37..95530b79 100644 --- a/libraries/chain/db_init.cpp +++ b/libraries/chain/db_init.cpp @@ -151,7 +151,7 @@ void database::init_genesis(const genesis_allocation& initial_allocation) n.owner.add_authority(genesis_key.get_id(), 1); n.owner.weight_threshold = 1; n.active = n.owner; - n.memo_key = genesis_key.id; + n.options.memo_key = genesis_key.id; n.statistics = genesis_statistics.id; }); @@ -262,7 +262,7 @@ void database::init_genesis(const genesis_allocation& initial_allocation) auto mangle_to_name = [](const fc::static_variant& key) { string addr = string(key.which() == std::decay::type::tag
::value? key.get
() : key.get()); - string result = "bts"; + string result = "import"; string key_string = string(addr).substr(sizeof(GRAPHENE_ADDRESS_PREFIX)-1); for( char c : key_string ) { @@ -295,7 +295,7 @@ void database::init_genesis(const genesis_allocation& initial_allocation) cop.registrar = account_id_type(1); cop.active = account_authority; cop.owner = account_authority; - cop.memo_key = key_id; + cop.options.memo_key = key_id; trx.operations.push_back(cop); trx.validate(); auto ptrx = apply_transaction(trx, ~0); diff --git a/libraries/chain/db_maint.cpp b/libraries/chain/db_maint.cpp index 23f16537..55a531ab 100644 --- a/libraries/chain/db_maint.cpp +++ b/libraries/chain/db_maint.cpp @@ -307,15 +307,16 @@ void database::perform_chain_maintenance(const signed_block& next_block, const g // Usually they're the same, but if the stake account has specified a voting_account, that account is the one // specifying the opinions. const account_object& opinion_account = - (stake_account.voting_account == account_id_type())? stake_account - : d.get(stake_account.voting_account); + (stake_account.options.voting_account == + account_id_type())? stake_account + : d.get(stake_account.options.voting_account); const auto& stats = stake_account.statistics(d); uint64_t voting_stake = stats.total_core_in_orders.value + (stake_account.cashback_vb.valid() ? (*stake_account.cashback_vb)(d).balance.amount.value: 0) + d.get_balance(stake_account.get_id(), asset_id_type()).amount.value; - for( vote_id_type id : opinion_account.votes ) + for( vote_id_type id : opinion_account.options.votes ) { uint32_t offset = id.instance(); // if they somehow managed to specify an illegal offset, ignore it. @@ -323,9 +324,9 @@ void database::perform_chain_maintenance(const signed_block& next_block, const g d._vote_tally_buffer[ offset ] += voting_stake; } - if( opinion_account.num_witness <= props.parameters.maximum_witness_count ) + if( opinion_account.options.num_witness <= props.parameters.maximum_witness_count ) { - uint16_t offset = std::min(size_t(opinion_account.num_witness/2), + uint16_t offset = std::min(size_t(opinion_account.options.num_witness/2), d._witness_count_histogram_buffer.size() - 1); // votes for a number greater than maximum_witness_count // are turned into votes for maximum_witness_count. @@ -335,9 +336,9 @@ void database::perform_chain_maintenance(const signed_block& next_block, const g // parameter was lowered. d._witness_count_histogram_buffer[ offset ] += voting_stake; } - if( opinion_account.num_committee <= props.parameters.maximum_committee_count ) + if( opinion_account.options.num_committee <= props.parameters.maximum_committee_count ) { - uint16_t offset = std::min(size_t(opinion_account.num_committee/2), + uint16_t offset = std::min(size_t(opinion_account.options.num_committee/2), d._committee_count_histogram_buffer.size() - 1); // votes for a number greater than maximum_committee_count // are turned into votes for maximum_committee_count. diff --git a/libraries/chain/evaluator.cpp b/libraries/chain/evaluator.cpp index 46117211..e390890e 100644 --- a/libraries/chain/evaluator.cpp +++ b/libraries/chain/evaluator.cpp @@ -115,7 +115,15 @@ namespace graphene { namespace chain { return rel_id; } - authority generic_evaluator::resolve_relative_ids( const authority& a )const + void generic_evaluator::check_relative_ids(const authority& a)const + { + for( const auto& item : a.auths ) + { + auto id = get_relative_id( item.first ); + FC_ASSERT( id.type() == key_object_type || id.type() == account_object_type ); + } + } + authority generic_evaluator::resolve_relative_ids(const authority& a)const { authority result; result.auths.reserve( a.auths.size() ); diff --git a/libraries/chain/include/graphene/chain/account_object.hpp b/libraries/chain/include/graphene/chain/account_object.hpp index 0a3497d6..c1d7376a 100644 --- a/libraries/chain/include/graphene/chain/account_object.hpp +++ b/libraries/chain/include/graphene/chain/account_object.hpp @@ -144,6 +144,7 @@ class database; /// The account's name. This name must be unique among all account names on the graph. May not be empty. string name; + /** * The owner authority represents absolute control over the account. Usually the keys in this authority will * be kept in cold storage, as they should not be needed very often and compromise of these keys constitutes @@ -154,19 +155,30 @@ class database; /// The owner authority contains the hot keys of the account. This authority has control over nearly all /// operations the account may perform. authority active; - /// The memo key is the key this account will typically use to encrypt/sign transaction memos and other non- - /// validated account activities. This field is here to prevent confusion if the active authority has zero or - /// multiple keys in it. - key_id_type memo_key; - /// If this field is set to an account ID other than 0, this account's votes will be ignored and its stake - /// will be counted as voting for the referenced account's selected votes instead. - account_id_type voting_account; - uint16_t num_witness = 0; - uint16_t num_committee = 0; - /// This is the list of vote IDs this account votes for. The weight of these votes is determined by this - /// account's balance of core asset. - flat_set votes; + /// These are the fields which can be updated by the active authority. + struct options_type { + /// The memo key is the key this account will typically use to encrypt/sign transaction memos and other non- + /// validated account activities. This field is here to prevent confusion if the active authority has zero or + /// multiple keys in it. + object_id_type memo_key = key_id_type(); + key_id_type get_memo_key()const { return memo_key; } + /// If this field is set to an account ID other than 0, this account's votes will be ignored and its stake + /// will be counted as voting for the referenced account's selected votes instead. + account_id_type voting_account; + + /// The number of active witnesses this account votes the blockchain should appoint + /// Must not exceed the actual number of witnesses voted for in @ref votes + uint16_t num_witness = 0; + /// The number of active committee members this account votes the blockchain should appoint + /// Must not exceed the actual number of committee members voted for in @ref votes + uint16_t num_committee = 0; + /// This is the list of vote IDs this account votes for. The weight of these votes is determined by this + /// account's balance of core asset. + flat_set votes; + + void validate()const; + } options; /// The reference implementation records the account's statistics in a separate object. This field contains the /// ID of that object. @@ -293,8 +305,10 @@ FC_REFLECT_DERIVED( graphene::chain::account_object, (graphene::db::annotated_object), (membership_expiration_date)(registrar)(referrer)(lifetime_referrer) (network_fee_percentage)(lifetime_referrer_fee_percentage)(referrer_rewards_percentage) - (name)(owner)(active)(memo_key)(voting_account)(num_witness)(num_committee)(votes) - (statistics)(whitelisting_accounts)(blacklisting_accounts)(cashback_vb) ) + (name)(owner)(active)(options)(statistics)(whitelisting_accounts)(blacklisting_accounts) + (cashback_vb) ) + +FC_REFLECT(graphene::chain::account_object::options_type, (memo_key)(voting_account)(num_witness)(num_committee)(votes)) FC_REFLECT_DERIVED( graphene::chain::account_balance_object, (graphene::db::object), diff --git a/libraries/chain/include/graphene/chain/config.hpp b/libraries/chain/include/graphene/chain/config.hpp index 507b5641..066a18e6 100644 --- a/libraries/chain/include/graphene/chain/config.hpp +++ b/libraries/chain/include/graphene/chain/config.hpp @@ -112,7 +112,6 @@ #define GRAPHENE_DEFAULT_WORKER_BUDGET_PER_DAY (GRAPHENE_BLOCKCHAIN_PRECISION * int64_t(500) * 1000 ) #define GRAPHENE_MAX_INTEREST_APR uint16_t( 10000 ) -#define GRAPHENE_LEGACY_NAME_IMPORT_PERIOD 3000000 /** 3 million blocks */ /** * Reserved Account IDs with special meaning diff --git a/libraries/chain/include/graphene/chain/evaluator.hpp b/libraries/chain/include/graphene/chain/evaluator.hpp index 922a056d..4e3996d4 100644 --- a/libraries/chain/include/graphene/chain/evaluator.hpp +++ b/libraries/chain/include/graphene/chain/evaluator.hpp @@ -102,6 +102,7 @@ namespace graphene { namespace chain { object_id_type get_relative_id( object_id_type rel_id )const; + void check_relative_ids(const authority& a)const; authority resolve_relative_ids( const authority& a )const; asset fee_from_account; diff --git a/libraries/chain/include/graphene/chain/operations.hpp b/libraries/chain/include/graphene/chain/operations.hpp index 7be5368a..5d5f787c 100644 --- a/libraries/chain/include/graphene/chain/operations.hpp +++ b/libraries/chain/include/graphene/chain/operations.hpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -137,12 +138,8 @@ namespace graphene { namespace chain { string name; authority owner; authority active; - account_id_type voting_account; - object_id_type memo_key = key_id_type(); - uint16_t num_witness = 0; - uint16_t num_committee = 0; - flat_set vote; + account_object::options_type options; account_id_type fee_payer()const { return registrar; } void get_required_auth(flat_set& active_auth_set , flat_set&)const; @@ -203,15 +200,17 @@ namespace graphene { namespace chain { */ struct account_update_operation { - asset fee; - account_id_type account; - optional owner; - optional active; - optional voting_account; - optional memo_key; - optional> vote; - uint16_t num_witness = 0; - uint16_t num_committee = 0; + asset fee; + /// The account to update + account_id_type account; + + /// New owner authority. If set, this operation requires owner authority to execute. + optional owner; + /// New active authority. If set, this operation requires owner authority to execute. + optional active; + + /// New account options + optional new_options; account_id_type fee_payer()const { return account; } void get_required_auth(flat_set& active_auth_set , flat_set& owner_auth_set)const; @@ -1566,13 +1565,11 @@ FC_REFLECT( graphene::chain::key_create_operation, FC_REFLECT( graphene::chain::account_create_operation, (fee)(registrar) (referrer)(referrer_percent) - (name) - (owner)(active)(voting_account)(memo_key) - (num_witness)(num_committee)(vote) + (name)(owner)(active)(options) ) FC_REFLECT( graphene::chain::account_update_operation, - (fee)(account)(owner)(active)(voting_account)(memo_key)(num_witness)(num_committee)(vote) + (fee)(account)(owner)(active)(new_options) ) FC_REFLECT( graphene::chain::account_upgrade_operation, (fee)(account_to_upgrade)(upgrade_to_lifetime_member) ) diff --git a/libraries/chain/operations.cpp b/libraries/chain/operations.cpp index 5e88efed..a90ebb52 100644 --- a/libraries/chain/operations.cpp +++ b/libraries/chain/operations.cpp @@ -149,9 +149,11 @@ void account_update_operation::validate()const { FC_ASSERT( fee.amount >= 0 ); FC_ASSERT( account != account_id_type() ); - FC_ASSERT( owner || active || voting_account || memo_key || vote ); -} + FC_ASSERT( owner || active || new_options ); + if( new_options ) + new_options->validate(); +} share_type asset_create_operation::calculate_fee( const fee_schedule_type& schedule )const { @@ -209,10 +211,7 @@ void account_create_operation::validate()const FC_ASSERT( owner.weight_threshold == 1 ); FC_ASSERT( owner.auths.size() == 1 ); } - FC_ASSERT( num_witness + num_committee >= num_witness ); // no overflow - FC_ASSERT( num_witness + num_committee <= vote.size() ); - // FC_ASSERT( (num_witness == 0) || (num_witness&0x01) == 0, "must be odd number" ); - // FC_ASSERT( (num_committee == 0) || (num_committee&0x01) == 0, "must be odd number" ); + options.validate(); } diff --git a/libraries/plugins/account_history/account_history_plugin.cpp b/libraries/plugins/account_history/account_history_plugin.cpp index d8e1148e..1912cc84 100644 --- a/libraries/plugins/account_history/account_history_plugin.cpp +++ b/libraries/plugins/account_history/account_history_plugin.cpp @@ -448,7 +448,7 @@ flat_set account_history_plugin_impl::get_keys_for_account( const a flat_set key_id_set; key_id_set.reserve(owner_auths.size() + active_auths.size() + 2); - key_id_set.insert(acct.memo_key); + key_id_set.insert(acct.options.memo_key); // we don't use get_keys() here to avoid an intermediate copy operation for( const pair& item : active_auths ) diff --git a/libraries/utilities/git_revision.cpp.in b/libraries/utilities/git_revision.cpp.in index 411e454a..b615f1ba 100644 --- a/libraries/utilities/git_revision.cpp.in +++ b/libraries/utilities/git_revision.cpp.in @@ -5,10 +5,10 @@ #define GRAPHENE_GIT_REVISION_UNIX_TIMESTAMP @GRAPHENE_GIT_REVISION_UNIX_TIMESTAMP@ #define GRAPHENE_GIT_REVISION_DESCRIPTION "@GRAPHENE_GIT_REVISION_DESCRIPTION@" -namespace bts { namespace utilities { +namespace graphene { namespace utilities { const char* const git_revision_sha = GRAPHENE_GIT_REVISION_SHA; const uint32_t git_revision_unix_timestamp = GRAPHENE_GIT_REVISION_UNIX_TIMESTAMP; const char* const git_revision_description = GRAPHENE_GIT_REVISION_DESCRIPTION; -} } // end namespace bts::utilities +} } // end namespace graphene::utilities diff --git a/libraries/wallet/wallet.cpp b/libraries/wallet/wallet.cpp index 2d0d6dec..6dc9e67a 100644 --- a/libraries/wallet/wallet.cpp +++ b/libraries/wallet/wallet.cpp @@ -724,7 +724,7 @@ public: account_create_op.name = name; account_create_op.owner = authority(1, owner_rkid, 1); account_create_op.active = authority(1, active_rkid, 1); - account_create_op.memo_key = active_rkid; + account_create_op.options.memo_key = active_rkid; signed_transaction tx; @@ -820,7 +820,7 @@ public: account_create_op.name = account_name; account_create_op.owner = authority(1, owner_rkid, 1); account_create_op.active = authority(1, active_rkid, 1); - account_create_op.memo_key = active_rkid; + account_create_op.options.memo_key = active_rkid; // current_fee_schedule() // find_account(pay_from_account) @@ -1087,10 +1087,10 @@ public: if( memo.size() ) { xfer_op.memo = memo_data(); - xfer_op.memo->from = from_account.memo_key; - xfer_op.memo->to = to_account.memo_key; - xfer_op.memo->set_message(get_private_key(from_account.memo_key), - get_public_key(to_account.memo_key), memo); + xfer_op.memo->from = from_account.options.memo_key; + xfer_op.memo->to = to_account.options.memo_key; + xfer_op.memo->set_message(get_private_key(from_account.options.memo_key), + get_public_key(to_account.options.memo_key), memo); } signed_transaction tx; @@ -1117,10 +1117,10 @@ public: if( memo.size() ) { issue_op.memo = memo_data(); - issue_op.memo->from = issuer.memo_key; - issue_op.memo->to = to.memo_key; - issue_op.memo->set_message(get_private_key(issuer.memo_key), - get_public_key(to.memo_key), memo); + issue_op.memo->from = issuer.options.memo_key; + issue_op.memo->to = to.options.memo_key; + issue_op.memo->set_message(get_private_key(issuer.options.memo_key), + get_public_key(to.options.memo_key), memo); } signed_transaction tx; @@ -1200,7 +1200,7 @@ void dbg_make_mia(string creator, string symbol) void flood_network(string prefix, uint32_t number_of_transactions) { - const account_object& master = *_wallet.my_accounts.get().lower_bound("bts"); + const account_object& master = *_wallet.my_accounts.get().lower_bound("import"); int number_of_accounts = number_of_transactions / 3; number_of_transactions -= number_of_accounts; auto key = derive_private_key("floodshill", 0); diff --git a/tests/common/database_fixture.cpp b/tests/common/database_fixture.cpp index 61a5c8cf..d396826b 100644 --- a/tests/common/database_fixture.cpp +++ b/tests/common/database_fixture.cpp @@ -193,7 +193,7 @@ void database_fixture::verify_account_history_plugin_index( )const if( auth.first.type() == key_object_type ) acct_addresses.insert( key_id_type( auth.first )(db).key_address() ); } - acct_addresses.insert( acct.memo_key(db).key_address() ); + acct_addresses.insert( acct.options.get_memo_key()(db).key_address() ); for( const address& addr : acct_addresses ) tuples_from_db.emplace_back( account_id, addr ); } @@ -294,7 +294,7 @@ account_create_operation database_fixture::make_account( create_account.name = name; create_account.owner = authority(123, key, 123); create_account.active = authority(321, key, 321); - create_account.memo_key = key; + create_account.options.memo_key = key; auto& active_delegates = db.get_global_properties().active_delegates; if( active_delegates.size() > 0 ) @@ -305,9 +305,9 @@ account_create_operation database_fixture::make_account( votes.insert(active_delegates[rand() % active_delegates.size()](db).vote_id); votes.insert(active_delegates[rand() % active_delegates.size()](db).vote_id); votes.insert(active_delegates[rand() % active_delegates.size()](db).vote_id); - create_account.vote = flat_set(votes.begin(), votes.end()); + create_account.options.votes = flat_set(votes.begin(), votes.end()); } - create_account.num_committee = create_account.vote.size(); + create_account.options.num_committee = create_account.options.votes.size(); create_account.fee = create_account.calculate_fee(db.current_fee_schedule()); return create_account; @@ -332,7 +332,7 @@ account_create_operation database_fixture::make_account( create_account.name = name; create_account.owner = authority(123, key, 123); create_account.active = authority(321, key, 321); - create_account.memo_key = key; + create_account.options.memo_key = key; const vector& active_delegates = db.get_global_properties().active_delegates; if( active_delegates.size() > 0 ) @@ -343,9 +343,9 @@ account_create_operation database_fixture::make_account( votes.insert(active_delegates[rand() % active_delegates.size()](db).vote_id); votes.insert(active_delegates[rand() % active_delegates.size()](db).vote_id); votes.insert(active_delegates[rand() % active_delegates.size()](db).vote_id); - create_account.vote = flat_set(votes.begin(), votes.end()); + create_account.options.votes = flat_set(votes.begin(), votes.end()); } - create_account.num_committee = create_account.vote.size(); + create_account.options.num_committee = create_account.options.votes.size(); create_account.fee = create_account.calculate_fee(db.current_fee_schedule()); return create_account; @@ -499,7 +499,7 @@ const account_object& database_fixture::create_account( account_create_op.name = name; account_create_op.owner = authority(1234, key_rkid, 1234); account_create_op.active = authority(5678, key_rkid, 5678); - account_create_op.memo_key = key_rkid; + account_create_op.options.memo_key = key_rkid; trx.operations.push_back( account_create_op ); trx.validate(); diff --git a/tests/tests/authority_tests.cpp b/tests/tests/authority_tests.cpp index 03585f4b..4ce9740d 100644 --- a/tests/tests/authority_tests.cpp +++ b/tests/tests/authority_tests.cpp @@ -474,7 +474,8 @@ BOOST_FIXTURE_TEST_CASE( fired_delegates, database_fixture ) //Oh noes! Nathan votes for a whole new slate of delegates! account_update_operation op; op.account = nathan->id; - op.vote = delegates; + op.new_options = nathan->options; + op.new_options->votes = delegates; trx.operations.push_back(op); trx.set_expiration(db.head_block_time() + GRAPHENE_DEFAULT_MAX_TIME_UNTIL_EXPIRATION); db.push_transaction(trx, ~0); @@ -874,7 +875,7 @@ BOOST_FIXTURE_TEST_CASE( max_authority_membership, database_fixture ) anon_create_op.owner = owner_auth; anon_create_op.active = active_auth; anon_create_op.registrar = sam_account_object.id; - anon_create_op.memo_key = sam_account_object.memo_key; + anon_create_op.options.memo_key = sam_account_object.options.memo_key; anon_create_op.name = generate_anon_acct_name(); tx.operations.push_back( anon_create_op ); @@ -913,10 +914,10 @@ BOOST_FIXTURE_TEST_CASE( bogus_signature, database_fixture ) account_object bob_account_object = create_account( "bob", bob_key ); account_object charlie_account_object = create_account( "charlie", charlie_key ); - key_id_type alice_key_id = alice_account_object.memo_key; + key_id_type alice_key_id = alice_account_object.options.memo_key; // unneeded, comment it out to silence compiler warning //key_id_type bob_key_id = bob_account_object.memo_key; - key_id_type charlie_key_id = charlie_account_object.memo_key; + key_id_type charlie_key_id = charlie_account_object.options.memo_key; uint32_t skip = database::skip_transaction_dupe_check; @@ -994,9 +995,10 @@ BOOST_FIXTURE_TEST_CASE( voting_account, database_fixture ) { account_update_operation op; op.account = nathan_id; - op.voting_account = vikram_id; - op.vote = flat_set{nathan_delegate(db).vote_id}; - op.num_committee = 1; + op.new_options = nathan_id(db).options; + op.new_options->voting_account = vikram_id; + op.new_options->votes = flat_set{nathan_delegate(db).vote_id}; + op.new_options->num_committee = 1; trx.operations.push_back(op); trx.sign(nathan_key_id, nathan_private_key); db.push_transaction(trx); @@ -1005,11 +1007,16 @@ BOOST_FIXTURE_TEST_CASE( voting_account, database_fixture ) { account_update_operation op; op.account = vikram_id; - op.vote = vikram_id(db).votes; - op.vote->insert(vikram_delegate(db).vote_id); - op.num_committee = 11; + op.new_options = vikram_id(db).options; + op.new_options->votes.insert(vikram_delegate(db).vote_id); + op.new_options->num_committee = 11; trx.operations.push_back(op); trx.sign(vikram_key_id, vikram_private_key); + // Fails because num_committee is larger than the cardinality of committee members being voted for + BOOST_CHECK_THROW(db.push_transaction(trx), fc::exception); + op.new_options->num_committee = 3; + trx.operations = {op}; + trx.sign(vikram_key_id, vikram_private_key); db.push_transaction(trx); trx.clear(); } diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp index c74e182d..8704850a 100644 --- a/tests/tests/block_tests.cpp +++ b/tests/tests/block_tests.cpp @@ -429,8 +429,8 @@ BOOST_FIXTURE_TEST_CASE( maintenance_interval, database_fixture ) { account_update_operation op; op.account = nathan.id; - op.vote = nathan.votes; - op.vote->insert(nathans_delegate.vote_id); + op.new_options = nathan.options; + op.new_options->votes.insert(nathans_delegate.vote_id); trx.operations.push_back(op); db.push_transaction(trx, ~0); trx.operations.clear(); diff --git a/tests/tests/fee_tests.cpp b/tests/tests/fee_tests.cpp index e72ca4d0..f5b429d7 100644 --- a/tests/tests/fee_tests.cpp +++ b/tests/tests/fee_tests.cpp @@ -71,7 +71,7 @@ BOOST_AUTO_TEST_CASE( cashback_test ) op.referrer = referrer_name ## _id; \ op.referrer_percent = referrer_rate*GRAPHENE_1_PERCENT; \ op.name = BOOST_PP_STRINGIZE(actor_name); \ - op.memo_key = actor_name ## _key_id; \ + op.options.memo_key = actor_name ## _key_id; \ op.active = authority(1, actor_name ## _key_id, 1); \ op.owner = op.active; \ op.fee = op.calculate_fee(fees); \ diff --git a/tests/tests/operation_tests.cpp b/tests/tests/operation_tests.cpp index e3bc8c00..f2c503be 100644 --- a/tests/tests/operation_tests.cpp +++ b/tests/tests/operation_tests.cpp @@ -57,8 +57,8 @@ BOOST_AUTO_TEST_CASE( create_account_test ) REQUIRE_THROW_WITH_VALUE(op, name, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); REQUIRE_THROW_WITH_VALUE(op, name, "aaaa."); REQUIRE_THROW_WITH_VALUE(op, name, ".aaaa"); - REQUIRE_THROW_WITH_VALUE(op, voting_account, account_id_type(999999999)); - REQUIRE_THROW_WITH_VALUE(op, memo_key, key_id_type(999999999)); + REQUIRE_THROW_WITH_VALUE(op, options.voting_account, account_id_type(999999999)); + REQUIRE_THROW_WITH_VALUE(op, options.memo_key, key_id_type(999999999)); auto auth_bak = op.owner; op.owner.add_authority(account_id_type(9999999999), 10); @@ -85,8 +85,8 @@ BOOST_AUTO_TEST_CASE( create_account_test ) BOOST_CHECK(nathan_account.owner.auths.at(genesis_key) == 123); BOOST_REQUIRE(nathan_account.active.auths.size() == 1); BOOST_CHECK(nathan_account.active.auths.at(genesis_key) == 321); - BOOST_CHECK(nathan_account.voting_account == account_id_type()); - BOOST_CHECK(nathan_account.memo_key == genesis_key); + BOOST_CHECK(nathan_account.options.voting_account == account_id_type()); + BOOST_CHECK(nathan_account.options.memo_key == genesis_key); const account_statistics_object& statistics = nathan_account.statistics(db); BOOST_CHECK(statistics.id.space() == implementation_ids); @@ -159,13 +159,13 @@ BOOST_AUTO_TEST_CASE( update_account ) op.account = nathan.id; op.owner = authority(2, key_id, 1, key_id_type(), 1); op.active = authority(2, key_id, 1, key_id_type(), 1); - //op.voting_account = key_id; - op.vote = flat_set({active_delegates[0](db).vote_id, active_delegates[5](db).vote_id}); + op.new_options = nathan.options; + op.new_options->votes = flat_set({active_delegates[0](db).vote_id, active_delegates[5](db).vote_id}); + op.new_options->num_committee = 2; trx.operations.back() = op; db.push_transaction(trx, ~0); - //BOOST_CHECK(nathan.voting_key == key_id); - BOOST_CHECK(nathan.memo_key == key_id_type()); + BOOST_CHECK(nathan.options.memo_key == key_id_type()); BOOST_CHECK(nathan.active.weight_threshold == 2); BOOST_CHECK(nathan.active.auths.size() == 2); BOOST_CHECK(nathan.active.auths.at(key_id) == 1); @@ -174,7 +174,7 @@ BOOST_AUTO_TEST_CASE( update_account ) BOOST_CHECK(nathan.owner.auths.size() == 2); BOOST_CHECK(nathan.owner.auths.at(key_id) == 1); BOOST_CHECK(nathan.owner.auths.at(key_id_type()) == 1); - BOOST_CHECK(nathan.votes.size() == 2); + BOOST_CHECK(nathan.options.votes.size() == 2); /** these votes are no longer tallied in real time BOOST_CHECK(active_delegates[0](db).vote(db).total_votes == 30000); diff --git a/tests/tests/operation_tests2.cpp b/tests/tests/operation_tests2.cpp index 67f11d19..7e72bbbf 100644 --- a/tests/tests/operation_tests2.cpp +++ b/tests/tests/operation_tests2.cpp @@ -399,10 +399,12 @@ BOOST_AUTO_TEST_CASE( witness_create ) { account_update_operation op; op.account = nathan_id; - op.vote = nathan_id(db).votes; - op.vote->insert(nathan_witness_id(db).vote_id); - op.num_witness = std::count_if(op.vote->begin(), op.vote->end(), [](vote_id_type id) { return id.type() == vote_id_type::witness; }); - op.num_committee = std::count_if(op.vote->begin(), op.vote->end(), [](vote_id_type id) { return id.type() == vote_id_type::committee; }); + op.new_options = nathan_id(db).options; + op.new_options->votes.insert(nathan_witness_id(db).vote_id); + op.new_options->num_witness = std::count_if(op.new_options->votes.begin(), op.new_options->votes.end(), + [](vote_id_type id) { return id.type() == vote_id_type::witness; }); + op.new_options->num_committee = std::count_if(op.new_options->votes.begin(), op.new_options->votes.end(), + [](vote_id_type id) { return id.type() == vote_id_type::committee; }); trx.operations.push_back(op); trx.sign(nathan_key_id, nathan_private_key); db.push_transaction(trx); @@ -540,8 +542,8 @@ BOOST_AUTO_TEST_CASE( worker_pay_test ) { account_update_operation op; op.account = nathan_id; - op.vote = nathan_id(db).votes; - op.vote->insert(worker_id_type()(db).vote_for); + op.new_options = nathan_id(db).options; + op.new_options->votes.insert(worker_id_type()(db).vote_for); trx.operations.push_back(op); db.push_transaction(trx, ~0); trx.clear(); @@ -580,8 +582,8 @@ BOOST_AUTO_TEST_CASE( worker_pay_test ) { account_update_operation op; op.account = nathan_id; - op.vote = nathan_id(db).votes; - op.vote->erase(worker_id_type()(db).vote_for); + op.new_options = nathan_id(db).options; + op.new_options->votes.erase(worker_id_type()(db).vote_for); trx.operations.push_back(op); db.push_transaction(trx, ~0); trx.clear(); @@ -653,8 +655,8 @@ BOOST_AUTO_TEST_CASE( refund_worker_test ) { account_update_operation op; op.account = nathan_id; - op.vote = nathan_id(db).votes; - op.vote->insert(worker_id_type()(db).vote_for); + op.new_options = nathan_id(db).options; + op.new_options->votes.insert(worker_id_type()(db).vote_for); trx.operations.push_back(op); db.push_transaction(trx, ~0); trx.clear(); From ca89d5057c69dab4dd0a88b441cf35996ad9e18d Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Tue, 16 Jun 2015 15:56:13 -0400 Subject: [PATCH 5/7] remove dependency on level db, bugs left to fix --- .gitmodules | 4 - CMakeLists.txt | 28 -- libraries/app/include/graphene/app/plugin.hpp | 1 + libraries/chain/CMakeLists.txt | 3 +- libraries/chain/block_database.cpp | 159 ++++++ libraries/chain/db_block.cpp | 14 +- libraries/chain/db_management.cpp | 17 +- libraries/chain/db_update.cpp | 2 + .../include/graphene/chain/block_database.hpp | 26 + .../chain/include/graphene/chain/database.hpp | 5 +- libraries/db/CMakeLists.txt | 4 +- libraries/db/include/graphene/db/index.hpp | 52 +- .../db/include/graphene/db/level_map.hpp | 462 ------------------ .../db/include/graphene/db/level_pod_map.hpp | 309 ------------ .../include/graphene/db/object_database.hpp | 4 - libraries/db/object_database.cpp | 62 +-- libraries/db/upgrade_leveldb.cpp | 114 ----- libraries/leveldb | 1 - libraries/net/CMakeLists.txt | 2 +- libraries/net/peer_database.cpp | 22 +- tests/common/database_fixture.hpp | 1 + 21 files changed, 274 insertions(+), 1018 deletions(-) create mode 100644 libraries/chain/block_database.cpp create mode 100644 libraries/chain/include/graphene/chain/block_database.hpp delete mode 100644 libraries/db/include/graphene/db/level_map.hpp delete mode 100644 libraries/db/include/graphene/db/level_pod_map.hpp delete mode 100644 libraries/db/upgrade_leveldb.cpp delete mode 160000 libraries/leveldb diff --git a/.gitmodules b/.gitmodules index 85a89fba..ee296306 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,7 +5,3 @@ path = libraries/fc url = https://github.com/cryptonomex/fc.git ignore = dirty -[submodule "libraries/leveldb"] - path = libraries/leveldb - url = https://github.com/bitcoin/leveldb.git - ignore = dirty diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c6e7652..d60399d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,21 +65,6 @@ IF(NOT "${Boost_VERSION}" MATCHES "1.53(.*)") SET(Boost_LIBRARIES ${BOOST_LIBRARIES_TEMP} ${Boost_LIBRARIES}) ENDIF() -set( LEVEL_DB_DIR "${CMAKE_SOURCE_DIR}/libraries/leveldb" ) - -file( GLOB LEVEL_DB_SOURCES "${LEVEL_DB_DIR}/db/*.cc" - "${LEVEL_DB_DIR}/helpers/memenv/memenv.cc" - "${LEVEL_DB_DIR}/table/*.cc" - "${LEVEL_DB_DIR}/util/*.cc" ) -foreach( filename ${LEVEL_DB_SOURCES} ) - if( ${filename} MATCHES ".*_test.cc" OR ${filename} MATCHES ".*_bench.cc" OR ${filename} MATCHES ".*_main.cc" ) - list( REMOVE_ITEM LEVEL_DB_SOURCES ${filename} ) - endif() -endforeach() -set(LEVELDB_BUILD_DEFINES) -set(LEVELDB_BUILD_LIBRARIES) -set(LEVELDB_BUILD_PRIVATE_INCLUDES "${LEVEL_DB_DIR}") - if( WIN32 ) message( STATUS "Configuring Graphene on WIN32") set( DB_VERSION 60 ) @@ -112,13 +97,7 @@ if( WIN32 ) SET(TCL_LIBS "${TCL_LIBS}${TCL_LIB_PATH}/${TCL_LIB_NAME}g${TCL_LIB_EXT}") SET(TCL_LIBRARY ${TCL_LIBS}) - SET(LEVELDB_PORT_FILE "${LEVEL_DB_DIR}/port/port_win.cc" ) - list(APPEND LEVELDB_BUILD_DEFINES OS_WINDOWS LEVELDB_PLATFORM_WINDOWS ) - list(APPEND LEVELDB_BUILD_LIBRARIES shlwapi.lib) - list(INSERT LEVELDB_BUILD_PRIVATE_INCLUDES 0 "${CMAKE_SOURCE_DIR}/libraries/leveldb-msvc/include") else( WIN32 ) # Apple AND Linux - SET(LEVELDB_PORT_FILE "${LEVEL_DB_DIR}/port/port_posix.cc" ) - list(APPEND LEVELDB_BUILD_DEFINES LEVELDB_PLATFORM_POSIX LEVELDB_ATOMIC_PRESENT) if( APPLE ) list(APPEND LEVELDB_BUILD_DEFINES OS_MACOSX) @@ -169,13 +148,6 @@ else( WIN32 ) # Apple AND Linux endif( WIN32 ) -list(APPEND LEVEL_DB_SOURCES "${LEVELDB_PORT_FILE}") -add_library( leveldb ${LEVEL_DB_SOURCES} ) -target_link_libraries( leveldb ${LEVELDB_BUILD_LIBRARIES} ) -target_include_directories( leveldb PRIVATE ${LEVELDB_BUILD_PRIVATE_INCLUDES} - PUBLIC "${LEVEL_DB_DIR}/include" ) -set_target_properties(leveldb PROPERTIES COMPILE_DEFINITIONS "${LEVELDB_BUILD_DEFINES}") - find_package( BerkeleyDB ) set(ENABLE_COVERAGE_TESTING FALSE CACHE BOOL "Build Graphene for code coverage analysis") diff --git a/libraries/app/include/graphene/app/plugin.hpp b/libraries/app/include/graphene/app/plugin.hpp index 253492ad..5065679d 100644 --- a/libraries/app/include/graphene/app/plugin.hpp +++ b/libraries/app/include/graphene/app/plugin.hpp @@ -20,6 +20,7 @@ #include #include +#include namespace graphene { namespace app { namespace bpo = boost::program_options; diff --git a/libraries/chain/CMakeLists.txt b/libraries/chain/CMakeLists.txt index 17635897..47c05498 100644 --- a/libraries/chain/CMakeLists.txt +++ b/libraries/chain/CMakeLists.txt @@ -34,6 +34,7 @@ add_library( graphene_chain transaction_evaluation_state.cpp fork_database.cpp + block_database.cpp db_balance.cpp db_block.cpp @@ -49,7 +50,7 @@ add_library( graphene_chain ${HEADERS} ) -target_link_libraries( graphene_chain fc graphene_db leveldb ) +target_link_libraries( graphene_chain fc graphene_db ) target_include_directories( graphene_chain PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) diff --git a/libraries/chain/block_database.cpp b/libraries/chain/block_database.cpp new file mode 100644 index 00000000..94389571 --- /dev/null +++ b/libraries/chain/block_database.cpp @@ -0,0 +1,159 @@ +#include + +namespace graphene { namespace chain { + +struct index_entry +{ + uint64_t block_pos = 0; + uint32_t block_size = 0; + block_id_type block_id; +}; + +void block_database::open( const fc::path& dbdir ) +{ + _block_num_to_pos.open( (dbdir/"index").generic_string().c_str(), std::fstream::binary ); + _blocks.open( (dbdir/"blocks").generic_string().c_str(), std::fstream::binary ); +} + + +bool block_database::is_open()const +{ + return _blocks.is_open(); +} + + +void block_database::close() +{ + _blocks.close(); + _block_num_to_pos.close(); +} + +void block_database::flush() +{ + _blocks.flush(); + _block_num_to_pos.flush(); +} + + +void block_database::store( const block_id_type& id, const signed_block& b ) +{ + auto num = block_header::num_from_id(id); + _block_num_to_pos.seekp( sizeof( index_entry ) * num ); + index_entry e; + _blocks.seekp( 0, _blocks.end ); + auto vec = fc::raw::pack( b ); + e.block_pos = _blocks.tellp(); + e.block_size = vec.size(); + e.block_id = id; + _blocks.write( vec.data(), vec.size() ); + _block_num_to_pos.write( (char*)&e, sizeof(e) ); +} + + +void block_database::remove( const block_id_type& id ) +{ + index_entry e; + auto index_pos = sizeof(e)*block_header::num_from_id(id); + _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); + FC_ASSERT( _block_num_to_pos.tellg() > index_pos ); + + _block_num_to_pos.seekg( index_pos ); + _block_num_to_pos.read( (char*)&e, sizeof(e) ); + + FC_ASSERT( e.block_id == id ); + + e.block_size = 0; + _block_num_to_pos.seekp( sizeof(e)*block_header::num_from_id(id) ); + _block_num_to_pos.write( (char*)&e, sizeof(e) ); +} + + + + + +bool block_database::contains( const block_id_type& id )const +{ + index_entry e; + auto index_pos = sizeof(e)*block_header::num_from_id(id); + _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); + FC_ASSERT( _block_num_to_pos.tellg() > index_pos ); + + _block_num_to_pos.seekg( index_pos ); + _block_num_to_pos.read( (char*)&e, sizeof(e) ); + + return e.block_id == id; +} + + +block_id_type block_database::fetch_block_id( uint32_t block_num )const +{ + index_entry e; + auto index_pos = sizeof(e)*block_num; + _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); + FC_ASSERT( _block_num_to_pos.tellg() > index_pos ); + + _block_num_to_pos.seekg( index_pos ); + _block_num_to_pos.read( (char*)&e, sizeof(e) ); + + return e.block_id; +} + + +optional block_database::fetch_optional( const block_id_type& id )const +{ + index_entry e; + auto index_pos = sizeof(e)*block_header::num_from_id(id); + _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); + FC_ASSERT( _block_num_to_pos.tellg() > index_pos ); + + _block_num_to_pos.seekg( index_pos ); + _block_num_to_pos.read( (char*)&e, sizeof(e) ); + + if( e.block_id != id ) return optional(); + + vector data( e.block_size ); + _blocks.seekg( e.block_pos ); + _blocks.read( data.data(), e.block_size ); + return fc::raw::unpack(data); +} + + +optional block_database::fetch_by_number( uint32_t block_num )const +{ + index_entry e; + auto index_pos = sizeof(e)*block_num; + _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); + FC_ASSERT( _block_num_to_pos.tellg() > index_pos ); + + _block_num_to_pos.seekg( index_pos ); + _block_num_to_pos.read( (char*)&e, sizeof(e) ); + + vector data( e.block_size ); + _blocks.seekg( e.block_pos ); + _blocks.read( data.data(), e.block_size ); + return fc::raw::unpack(data); +} + + +optional block_database::last()const +{ + index_entry e; + _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); + + if( _block_num_to_pos.tellp() < sizeof(index_entry) ) + return optional(); + + _block_num_to_pos.seekg( -sizeof(index_entry), _block_num_to_pos.end ); + _block_num_to_pos.read( (char*)&e, sizeof(e) ); + + if( e.block_size == 0 ) + return optional(); + + vector data( e.block_size ); + _blocks.seekg( e.block_pos ); + _blocks.read( data.data(), e.block_size ); + return fc::raw::unpack(data); +} + + +} } diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp index cfdd63f8..e8493820 100644 --- a/libraries/chain/db_block.cpp +++ b/libraries/chain/db_block.cpp @@ -30,7 +30,7 @@ namespace graphene { namespace chain { bool database::is_known_block( const block_id_type& id )const { - return _fork_db.is_known_block(id) || _block_id_to_block.find(id).valid(); + return _fork_db.is_known_block(id) || _block_id_to_block.contains(id); } /** * Only return true *if* the transaction has not expired or been invalidated. If this @@ -45,10 +45,7 @@ bool database::is_known_transaction( const transaction_id_type& id )const block_id_type database::get_block_id_for_num( uint32_t block_num )const { try { - block_id_type lb; lb._hash[0] = htonl(block_num); - auto itr = _block_id_to_block.lower_bound( lb ); - FC_ASSERT( itr.valid() && itr.key()._hash[0] == lb._hash[0] ); - return itr.key(); + return _block_id_to_block.fetch_block_id( block_num ); } FC_CAPTURE_AND_RETHROW( (block_num) ) } optional database::fetch_block_by_id( const block_id_type& id )const @@ -65,12 +62,7 @@ optional database::fetch_block_by_number( uint32_t num )const if( results.size() == 1 ) return results[0]->data; else - { - block_id_type lb; lb._hash[0] = htonl(num); - auto itr = _block_id_to_block.lower_bound( lb ); - if( itr.valid() && itr.key()._hash[0] == lb._hash[0] ) - return itr.value(); - } + return _block_id_to_block.fetch_by_number(num); return optional(); } diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp index de45ecda..5651fe1e 100644 --- a/libraries/chain/db_management.cpp +++ b/libraries/chain/db_management.cpp @@ -46,9 +46,9 @@ void database::open( const fc::path& data_dir, const genesis_allocation& initial _pending_block.previous = head_block_id(); _pending_block.timestamp = head_block_time(); - auto last_block_itr = _block_id_to_block.last(); - if( last_block_itr.valid() ) - _fork_db.start_block( last_block_itr.value() ); + auto last_block= _block_id_to_block.last(); + if( last_block.valid() ) + _fork_db.start_block( *last_block ); } FC_CAPTURE_AND_RETHROW( (data_dir) ) } @@ -58,19 +58,22 @@ void database::reindex(fc::path data_dir, const genesis_allocation& initial_allo open(data_dir, initial_allocation); auto start = fc::time_point::now(); - auto itr = _block_id_to_block.begin(); + auto last_block = _block_id_to_block.last(); + if( !last_block ) return; + + const auto last_block_num = last_block->block_num(); + // TODO: disable undo tracking durring reindex, this currently causes crashes in the benchmark test //_undo_db.disable(); - while( itr.valid() ) + for( uint32_t i = 1; i <= last_block_num; ++i ) { - apply_block( itr.value(), skip_delegate_signature | + apply_block( *_block_id_to_block.fetch_by_number(i), skip_delegate_signature | skip_transaction_signatures | skip_undo_block | skip_undo_transaction | skip_transaction_dupe_check | skip_tapos_check | skip_authority_check ); - ++itr; } //_undo_db.enable(); auto end = fc::time_point::now(); diff --git a/libraries/chain/db_update.cpp b/libraries/chain/db_update.cpp index 1ba5b95e..35b265db 100644 --- a/libraries/chain/db_update.cpp +++ b/libraries/chain/db_update.cpp @@ -187,9 +187,11 @@ void database::clear_expired_orders() max_settlement_volume = mia_object.amount(mia.max_force_settlement_volume(mia_object.dynamic_data(*this).current_supply)); if( mia.force_settled_volume >= max_settlement_volume.amount ) { + /* ilog("Skipping force settlement in ${asset}; settled ${settled_volume} / ${max_volume}", ("asset", mia_object.symbol)("settlement_price_null",mia.current_feed.settlement_price.is_null()) ("settled_volume", mia.force_settled_volume)("max_volume", max_settlement_volume)); + */ if( next_asset() ) continue; break; diff --git a/libraries/chain/include/graphene/chain/block_database.hpp b/libraries/chain/include/graphene/chain/block_database.hpp new file mode 100644 index 00000000..66af6ece --- /dev/null +++ b/libraries/chain/include/graphene/chain/block_database.hpp @@ -0,0 +1,26 @@ +#pragma once +#include +#include + +namespace graphene { namespace chain { + class block_database + { + public: + void open( const fc::path& dbdir ); + bool is_open()const; + void flush(); + void close(); + + void store( const block_id_type& id, const signed_block& b ); + void remove( const block_id_type& id ); + + bool contains( const block_id_type& id )const; + block_id_type fetch_block_id( uint32_t block_num )const; + optional fetch_optional( const block_id_type& id )const; + optional fetch_by_number( uint32_t block_num )const; + optional last()const; + private: + mutable std::fstream _blocks; + mutable std::fstream _block_num_to_pos; + }; +} } diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp index 469a78ad..58e7827b 100644 --- a/libraries/chain/include/graphene/chain/database.hpp +++ b/libraries/chain/include/graphene/chain/database.hpp @@ -23,11 +23,10 @@ #include #include #include +#include #include #include -#include -#include #include #include @@ -376,7 +375,7 @@ namespace graphene { namespace chain { * until the fork is resolved. This should make maintaining * the fork tree relatively simple. */ - graphene::db::level_map _block_id_to_block; + block_database _block_id_to_block; /** * Contains the set of ops that are in the process of being applied from diff --git a/libraries/db/CMakeLists.txt b/libraries/db/CMakeLists.txt index 68407487..91e26cc6 100644 --- a/libraries/db/CMakeLists.txt +++ b/libraries/db/CMakeLists.txt @@ -1,4 +1,4 @@ file(GLOB HEADERS "include/graphene/db/*.hpp") -add_library( graphene_db undo_database.cpp index.cpp object_database.cpp upgrade_leveldb.cpp ${HEADERS} ) -target_link_libraries( graphene_db fc leveldb ) +add_library( graphene_db undo_database.cpp index.cpp object_database.cpp ${HEADERS} ) +target_link_libraries( graphene_db fc ) target_include_directories( graphene_db PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) diff --git a/libraries/db/include/graphene/db/index.hpp b/libraries/db/include/graphene/db/index.hpp index 2c21d392..af95414d 100644 --- a/libraries/db/include/graphene/db/index.hpp +++ b/libraries/db/include/graphene/db/index.hpp @@ -17,10 +17,13 @@ */ #pragma once #include -#include +#include +#include +#include namespace graphene { namespace db { class object_database; + using fc::path; /** * @class index_observer @@ -80,9 +83,12 @@ namespace graphene { namespace db { virtual const object& create( const std::function& constructor ) = 0; /** - * Opens the index loading objects from a level_db database + * Opens the index loading objects from a file */ - virtual void open( const shared_ptr >>& db ){} + virtual void open( const fc::path& db ) = 0; + virtual void save( const fc::path& db ) = 0; + + /** @return the object with id or nullptr if not found */ virtual const object* find( object_id_type id )const = 0; @@ -170,22 +176,40 @@ namespace graphene { namespace db { virtual void use_next_id()override { ++_next_id.number; } virtual void set_next_id( object_id_type id )override { _next_id = id; } + virtual void open( const path& db )override + { + if( !fc::exists( db ) ) return; + fc::file_mapping fm( db.generic_string().c_str(), fc::read_only ); + fc::mapped_region mr( fm, fc::read_only, 0, fc::file_size(db) ); + fc::datastream ds( (const char*)mr.get_address(), mr.get_size() ); + fc::raw::unpack(ds, _next_id); + try { + vector tmp; + while( true ) + { + fc::raw::unpack( ds, tmp ); + load( tmp ); + } + } catch ( const fc::exception& ){} + } + + virtual void save( const path& db ) override + { + std::ofstream out( db.generic_string(), std::ofstream::binary ); + out.write( (char*)&_next_id, sizeof(_next_id) ); + this->inspect_all_objects( [&]( const object& o ) { + auto vec = fc::raw::pack( static_cast(o) ); + auto packed_vec = fc::raw::pack( vec ); + out.write( packed_vec.data(), packed_vec.size() ); + }); + } + virtual const object& load( const std::vector& data )override { return DerivedIndex::insert( fc::raw::unpack( data ) ); } - virtual void open( const shared_ptr >>& db )override - { - auto first = object_id_type( DerivedIndex::object_type::space_id, DerivedIndex::object_type::type_id, 0 ); - auto last = object_id_type( DerivedIndex::object_type::space_id, DerivedIndex::object_type::type_id+1, 0 ); - auto itr = db->lower_bound( first ); - while( itr.valid() && itr.key() < last ) - { - load( itr.value() ); - ++itr; - } - } + virtual const object& create(const std::function& constructor )override { const auto& result = DerivedIndex::create( constructor ); diff --git a/libraries/db/include/graphene/db/level_map.hpp b/libraries/db/include/graphene/db/level_map.hpp deleted file mode 100644 index 55ecb99b..00000000 --- a/libraries/db/include/graphene/db/level_map.hpp +++ /dev/null @@ -1,462 +0,0 @@ -/* - * Copyright (c) 2015, Cryptonomex, Inc. - * All rights reserved. - * - * This source code is provided for evaluation in private test networks only, until September 8, 2015. After this date, this license expires and - * the code may not be used, modified or distributed for any purpose. Redistribution and use in source and binary forms, with or without modification, - * are permitted until September 8, 2015, provided that the following conditions are met: - * - * 1. The code and/or derivative works are used only for private test networks consisting of no more than 10 P2P nodes. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -#pragma once - -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include - -#include - -namespace graphene { namespace db { - - namespace ldb = leveldb; - - /** - * @brief implements a high-level API on top of Level DB that stores items using fc::raw / reflection - */ - template - class level_map - { - public: - void open( const fc::path& dir, bool create = true, size_t cache_size = 0 ) - { try { - FC_ASSERT( !is_open(), "Database is already open!" ); - - ldb::Options opts; - opts.comparator = &_comparer; - opts.create_if_missing = create; - opts.max_open_files = 64; - opts.compression = leveldb::kNoCompression; - - if( cache_size > 0 ) - { - opts.write_buffer_size = cache_size / 4; // up to two write buffers may be held in memory simultaneously - _cache.reset( leveldb::NewLRUCache( cache_size / 2 ) ); - opts.block_cache = _cache.get(); - } - - if( ldb::kMajorVersion > 1 || ( leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16 ) ) - { - // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error - // on corruption in later versions. - opts.paranoid_checks = true; - } - - _read_options.verify_checksums = true; - _iter_options.verify_checksums = true; - _iter_options.fill_cache = false; - _sync_options.sync = true; - - // Given path must exist to succeed toNativeAnsiPath - fc::create_directories( dir ); - std::string ldbPath = dir.to_native_ansi_path(); - - ldb::DB* ndb = nullptr; - const auto ntrxstat = ldb::DB::Open( opts, ldbPath.c_str(), &ndb ); - if( !ntrxstat.ok() ) - { - elog( "Failure opening database: ${db}\nStatus: ${msg}", ("db",dir)("msg",ntrxstat.ToString()) ); - FC_THROW_EXCEPTION( level_map_open_failure, "Failure opening database: ${db}\nStatus: ${msg}", - ("db",dir)("msg",ntrxstat.ToString()) ); - } - _db.reset( ndb ); - - try_upgrade_db( dir, ndb, fc::get_typename::name(), sizeof( Value ) ); - } FC_CAPTURE_AND_RETHROW( (dir)(create)(cache_size) ) } - - bool is_open()const - { - return !!_db; - } - - void close() - { - _db.reset(); - _cache.reset(); - } - - fc::optional fetch_optional( const Key& k )const - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - auto itr = find( k ); - if( itr.valid() ) return itr.value(); - return fc::optional(); - } FC_RETHROW_EXCEPTIONS( warn, "" ) } - - Value fetch( const Key& k ) - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - std::vector kslice = fc::raw::pack( k ); - ldb::Slice ks( kslice.data(), kslice.size() ); - std::string value; - auto status = _db->Get( _read_options, ks, &value ); - if( status.IsNotFound() ) - { - FC_THROW_EXCEPTION( fc::key_not_found_exception, "unable to find key ${key}", ("key",k) ); - } - if( !status.ok() ) - { - FC_THROW_EXCEPTION( level_map_failure, "database error: ${msg}", ("msg", status.ToString() ) ); - } - fc::datastream ds(value.c_str(), value.size()); - Value tmp; - fc::raw::unpack(ds, tmp); - return tmp; - } FC_RETHROW_EXCEPTIONS( warn, "failure fetching key ${key}", ("key",k) ); } - - class iterator - { - public: - iterator(){} - bool valid()const - { - return _it && _it->Valid(); - } - - Key key()const - { - Key tmp_key; - fc::datastream ds2( _it->key().data(), _it->key().size() ); - fc::raw::unpack( ds2, tmp_key ); - return tmp_key; - } - - Value value()const - { - Value tmp_val; - fc::datastream ds( _it->value().data(), _it->value().size() ); - fc::raw::unpack( ds, tmp_val ); - return tmp_val; - } - - iterator& operator++() { _it->Next(); return *this; } - iterator& operator--() { _it->Prev(); return *this; } - - protected: - friend class level_map; - iterator( ldb::Iterator* it ) - :_it(it){} - - std::shared_ptr _it; - }; - - iterator begin() const - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - iterator itr( _db->NewIterator( _iter_options ) ); - itr._it->SeekToFirst(); - - if( itr._it->status().IsNotFound() ) - { - FC_THROW_EXCEPTION( fc::key_not_found_exception, "" ); - } - if( !itr._it->status().ok() ) - { - FC_THROW_EXCEPTION( level_map_failure, "database error: ${msg}", ("msg", itr._it->status().ToString() ) ); - } - - if( itr.valid() ) - { - return itr; - } - return iterator(); - } FC_RETHROW_EXCEPTIONS( warn, "error seeking to first" ) } - - iterator find( const Key& key )const - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - ldb::Slice key_slice; - - /** avoid dynamic memory allocation at this step if possible, most - * keys should be relatively small in size and not require dynamic - * memory allocation to seralize the key. - */ - fc::array stack_buffer; - - size_t pack_size = fc::raw::pack_size(key); - if( pack_size <= stack_buffer.size() ) - { - fc::datastream ds( stack_buffer.data, stack_buffer.size() ); - fc::raw::pack( ds ,key ); - key_slice = ldb::Slice( stack_buffer.data, pack_size ); - } - else - { - auto kslice = fc::raw::pack( key ); - key_slice = ldb::Slice( kslice.data(), kslice.size() ); - } - - iterator itr( _db->NewIterator( _iter_options ) ); - itr._it->Seek( key_slice ); - if( itr.valid() && itr.key() == key ) - { - return itr; - } - return iterator(); - } FC_RETHROW_EXCEPTIONS( warn, "error finding ${key}", ("key",key) ) } - - iterator lower_bound( const Key& key )const - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - std::vector kslice = fc::raw::pack( key ); - ldb::Slice key_slice( kslice.data(), kslice.size() ); - - iterator itr( _db->NewIterator( _iter_options ) ); - itr._it->Seek( key_slice ); - return itr; - } FC_RETHROW_EXCEPTIONS( warn, "error finding ${key}", ("key",key) ) } - - iterator last( )const - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - iterator itr( _db->NewIterator( _iter_options ) ); - itr._it->SeekToLast(); - return itr; - } FC_RETHROW_EXCEPTIONS( warn, "error finding last" ) } - - bool last( Key& k ) - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - std::unique_ptr it( _db->NewIterator( _iter_options ) ); - FC_ASSERT( it != nullptr ); - it->SeekToLast(); - if( !it->Valid() ) - { - return false; - } - fc::datastream ds2( it->key().data(), it->key().size() ); - fc::raw::unpack( ds2, k ); - return true; - } FC_RETHROW_EXCEPTIONS( warn, "error reading last item from database" ); } - - bool last( Key& k, Value& v ) - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - std::unique_ptr it( _db->NewIterator( _iter_options ) ); - FC_ASSERT( it != nullptr ); - it->SeekToLast(); - if( !it->Valid() ) - { - return false; - } - fc::datastream ds( it->value().data(), it->value().size() ); - fc::raw::unpack( ds, v ); - - fc::datastream ds2( it->key().data(), it->key().size() ); - fc::raw::unpack( ds2, k ); - return true; - } FC_RETHROW_EXCEPTIONS( warn, "error reading last item from database" ); } - - /** this class allows batched, atomic database writes. - * usage: - * { - * write_batch batch = _db.create_batch(); - * batch.store(key1, value1); - * batch.store(key2, value2); - * } - * when the batch goes out of scope, the operations are commited to the database - */ - class write_batch - { - private: - leveldb::WriteBatch _batch; - level_map* _map = nullptr; - leveldb::WriteOptions _write_options; - - friend class level_map; - write_batch( level_map* map, bool sync = false ) : _map(map) - { - _write_options.sync = sync; - } - public: - ~write_batch() - { - try - { - commit(); - } - catch (const fc::canceled_exception&) - { - throw; - } - catch (const fc::exception&) - { - // we're in a destructor, nothing we can do... - } - } - - void commit() - { - try - { - FC_ASSERT(_map->is_open(), "Database is not open!"); - - ldb::Status status = _map->_db->Write( _write_options, &_batch ); - if (!status.ok()) - FC_THROW_EXCEPTION(level_map_failure, "database error while applying batch: ${msg}", ("msg", status.ToString())); - _batch.Clear(); - } - FC_RETHROW_EXCEPTIONS(warn, "error applying batch"); - } - - void abort() - { - _batch.Clear(); - } - - void store( const Key& k, const Value& v ) - { - std::vector kslice = fc::raw::pack(k); - ldb::Slice ks(kslice.data(), kslice.size()); - - auto vec = fc::raw::pack(v); - ldb::Slice vs(vec.data(), vec.size()); - - _batch.Put(ks, vs); - } - - void remove( const Key& k ) - { - std::vector kslice = fc::raw::pack(k); - ldb::Slice ks(kslice.data(), kslice.size()); - _batch.Delete(ks); - } - }; - - write_batch create_batch( bool sync = false ) - { - FC_ASSERT( is_open(), "Database is not open!" ); - return write_batch( this, sync ); - } - - void store(const Key& k, const Value& v, bool sync = false) - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - std::vector kslice = fc::raw::pack( k ); - ldb::Slice ks( kslice.data(), kslice.size() ); - - auto vec = fc::raw::pack(v); - ldb::Slice vs( vec.data(), vec.size() ); - - auto status = _db->Put( sync ? _sync_options : _write_options, ks, vs ); - if( !status.ok() ) - { - FC_THROW_EXCEPTION( level_map_failure, "database error: ${msg}", ("msg", status.ToString() ) ); - } - } FC_RETHROW_EXCEPTIONS( warn, "error storing ${key} = ${value}", ("key",k)("value",v) ); } - - void remove( const Key& k, bool sync = false ) - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - std::vector kslice = fc::raw::pack( k ); - ldb::Slice ks( kslice.data(), kslice.size() ); - auto status = _db->Delete( sync ? _sync_options : _write_options, ks ); - if( !status.ok() ) - { - FC_THROW_EXCEPTION( level_map_failure, "database error: ${msg}", ("msg", status.ToString() ) ); - } - } FC_RETHROW_EXCEPTIONS( warn, "error removing ${key}", ("key",k) ); } - - void export_to_json( const fc::path& path )const - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - FC_ASSERT( !fc::exists( path ) ); - - std::ofstream fs( path.string() ); - fs.write( "[\n", 2 ); - - auto iter = begin(); - while( iter.valid() ) - { - auto str = fc::json::to_pretty_string( std::make_pair( iter.key(), iter.value() ) ); - if( (++iter).valid() ) str += ","; - str += "\n"; - fs.write( str.c_str(), str.size() ); - } - - fs.write( "]", 1 ); - } FC_CAPTURE_AND_RETHROW( (path) ) } - - // note: this loops through all the items in the database, so it's not exactly fast. it's intended for debugging, nothing else. - size_t size() const - { - FC_ASSERT( is_open(), "Database is not open!" ); - - iterator it = begin(); - size_t count = 0; - while (it.valid()) - { - ++count; - ++it; - } - return count; - } - - private: - class key_compare : public leveldb::Comparator - { - public: - int Compare( const leveldb::Slice& a, const leveldb::Slice& b )const - { - Key ak,bk; - fc::datastream dsa( a.data(), a.size() ); - fc::raw::unpack( dsa, ak ); - fc::datastream dsb( b.data(), b.size() ); - fc::raw::unpack( dsb, bk ); - - if( ak < bk ) return -1; - if( ak == bk ) return 0; - return 1; - } - - const char* Name()const { return "key_compare"; } - void FindShortestSeparator( std::string*, const leveldb::Slice& )const{} - void FindShortSuccessor( std::string* )const{}; - }; - - std::unique_ptr _db; - std::unique_ptr _cache; - key_compare _comparer; - - ldb::ReadOptions _read_options; - ldb::ReadOptions _iter_options; - ldb::WriteOptions _write_options; - ldb::WriteOptions _sync_options; - }; - -} } // graphene::db diff --git a/libraries/db/include/graphene/db/level_pod_map.hpp b/libraries/db/include/graphene/db/level_pod_map.hpp deleted file mode 100644 index 2883356f..00000000 --- a/libraries/db/include/graphene/db/level_pod_map.hpp +++ /dev/null @@ -1,309 +0,0 @@ -/* - * Copyright (c) 2015, Cryptonomex, Inc. - * All rights reserved. - * - * This source code is provided for evaluation in private test networks only, until September 8, 2015. After this date, this license expires and - * the code may not be used, modified or distributed for any purpose. Redistribution and use in source and binary forms, with or without modification, - * are permitted until September 8, 2015, provided that the following conditions are met: - * - * 1. The code and/or derivative works are used only for private test networks consisting of no more than 10 P2P nodes. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -#pragma once - -#include -#include -#include - -#include -#include - -#include -#include -#include - -namespace graphene { namespace db { - - namespace ldb = leveldb; - - /** - * @brief implements a high-level API on top of Level DB that stores items using fc::raw / reflection - * @note Key must be a POD type - */ - template - class level_pod_map - { - public: - void open( const fc::path& dir, bool create = true, size_t cache_size = 0 ) - { try { - FC_ASSERT( !is_open(), "Database is already open!" ); - - ldb::Options opts; - opts.comparator = &_comparer; - opts.create_if_missing = create; - opts.max_open_files = 64; - opts.compression = leveldb::kNoCompression; - - if( cache_size > 0 ) - { - opts.write_buffer_size = cache_size / 4; // up to two write buffers may be held in memory simultaneously - _cache.reset( leveldb::NewLRUCache( cache_size / 2 ) ); - opts.block_cache = _cache.get(); - } - - if( ldb::kMajorVersion > 1 || ( leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16 ) ) - { - // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error - // on corruption in later versions. - opts.paranoid_checks = true; - } - - _read_options.verify_checksums = true; - _iter_options.verify_checksums = true; - _iter_options.fill_cache = false; - _sync_options.sync = true; - - // Given path must exist to succeed toNativeAnsiPath - fc::create_directories( dir ); - std::string ldbPath = dir.to_native_ansi_path(); - - ldb::DB* ndb = nullptr; - const auto ntrxstat = ldb::DB::Open( opts, ldbPath.c_str(), &ndb ); - if( !ntrxstat.ok() ) - { - elog( "Failure opening database: ${db}\nStatus: ${msg}", ("db",dir)("msg",ntrxstat.ToString()) ); - FC_THROW_EXCEPTION( level_pod_map_open_failure, "Failure opening database: ${db}\nStatus: ${msg}", - ("db",dir)("msg",ntrxstat.ToString()) ); - } - _db.reset( ndb ); - - try_upgrade_db( dir, ndb, fc::get_typename::name(), sizeof( Value ) ); - } FC_CAPTURE_AND_RETHROW( (dir)(create)(cache_size) ) } - - bool is_open()const - { - return !!_db; - } - - void close() - { - _db.reset(); - _cache.reset(); - } - - fc::optional fetch_optional( const Key& k ) - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - auto itr = find( k ); - if( itr.valid() ) return itr.value(); - return fc::optional(); - } FC_RETHROW_EXCEPTIONS( warn, "" ) } - - Value fetch( const Key& key ) - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - ldb::Slice key_slice( (char*)&key, sizeof(key) ); - std::string value; - auto status = _db->Get( _read_options, key_slice, &value ); - if( status.IsNotFound() ) - { - FC_THROW_EXCEPTION( fc::key_not_found_exception, "unable to find key ${key}", ("key",key) ); - } - if( !status.ok() ) - { - FC_THROW_EXCEPTION( level_pod_map_failure, "database error: ${msg}", ("msg", status.ToString() ) ); - } - fc::datastream datastream(value.c_str(), value.size()); - Value tmp; - fc::raw::unpack(datastream, tmp); - return tmp; - } FC_RETHROW_EXCEPTIONS( warn, "error fetching key ${key}", ("key",key) ); } - - class iterator - { - public: - iterator(){} - bool valid()const - { - return _it && _it->Valid(); - } - - Key key()const - { - FC_ASSERT( sizeof(Key) == _it->key().size() ); - return *((Key*)_it->key().data()); - } - - Value value()const - { - Value tmp_val; - fc::datastream ds( _it->value().data(), _it->value().size() ); - fc::raw::unpack( ds, tmp_val ); - return tmp_val; - } - - iterator& operator++() { _it->Next(); return *this; } - iterator& operator--() { _it->Prev(); return *this; } - - protected: - friend class level_pod_map; - iterator( ldb::Iterator* it ) - :_it(it){} - - std::shared_ptr _it; - }; - iterator begin() - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - iterator itr( _db->NewIterator( _iter_options ) ); - itr._it->SeekToFirst(); - - if( itr._it->status().IsNotFound() ) - { - FC_THROW_EXCEPTION( fc::key_not_found_exception, "" ); - } - if( !itr._it->status().ok() ) - { - FC_THROW_EXCEPTION( level_pod_map_failure, "database error: ${msg}", ("msg", itr._it->status().ToString() ) ); - } - - if( itr.valid() ) - { - return itr; - } - return iterator(); - } FC_RETHROW_EXCEPTIONS( warn, "error seeking to first" ) } - - iterator find( const Key& key ) - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - ldb::Slice key_slice( (char*)&key, sizeof(key) ); - iterator itr( _db->NewIterator( _iter_options ) ); - itr._it->Seek( key_slice ); - if( itr.valid() && itr.key() == key ) - { - return itr; - } - return iterator(); - } FC_RETHROW_EXCEPTIONS( warn, "error finding ${key}", ("key",key) ) } - - iterator lower_bound( const Key& key ) - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - ldb::Slice key_slice( (char*)&key, sizeof(key) ); - iterator itr( _db->NewIterator( _iter_options ) ); - itr._it->Seek( key_slice ); - if( itr.valid() ) - { - return itr; - } - return iterator(); - } FC_RETHROW_EXCEPTIONS( warn, "error finding ${key}", ("key",key) ) } - - bool last( Key& k ) - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - std::unique_ptr it( _db->NewIterator( _iter_options ) ); - FC_ASSERT( it != nullptr ); - it->SeekToLast(); - if( !it->Valid() ) - { - return false; - } - FC_ASSERT( sizeof( Key) == it->key().size() ); - k = *((Key*)it->key().data()); - return true; - } FC_RETHROW_EXCEPTIONS( warn, "error reading last item from database" ); } - - bool last( Key& k, Value& v ) - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - std::unique_ptr it( _db->NewIterator( _iter_options ) ); - FC_ASSERT( it != nullptr ); - it->SeekToLast(); - if( !it->Valid() ) - { - return false; - } - fc::datastream ds( it->value().data(), it->value().size() ); - fc::raw::unpack( ds, v ); - - FC_ASSERT( sizeof( Key) == it->key().size() ); - k = *((Key*)it->key().data()); - return true; - } FC_RETHROW_EXCEPTIONS( warn, "error reading last item from database" ); } - - void store( const Key& k, const Value& v, bool sync = false ) - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - ldb::Slice ks( (char*)&k, sizeof(k) ); - auto vec = fc::raw::pack(v); - ldb::Slice vs( vec.data(), vec.size() ); - - auto status = _db->Put( sync ? _sync_options : _write_options, ks, vs ); - if( !status.ok() ) - { - FC_THROW_EXCEPTION( level_pod_map_failure, "database error: ${msg}", ("msg", status.ToString() ) ); - } - } FC_RETHROW_EXCEPTIONS( warn, "error storing ${key} = ${value}", ("key",k)("value",v) ); } - - void remove( const Key& k, bool sync = false ) - { try { - FC_ASSERT( is_open(), "Database is not open!" ); - - ldb::Slice ks( (char*)&k, sizeof(k) ); - auto status = _db->Delete( sync ? _sync_options : _write_options, ks ); - if( status.IsNotFound() ) - { - FC_THROW_EXCEPTION( fc::key_not_found_exception, "unable to find key ${key}", ("key",k) ); - } - if( !status.ok() ) - { - FC_THROW_EXCEPTION( level_pod_map_failure, "database error: ${msg}", ("msg", status.ToString() ) ); - } - } FC_RETHROW_EXCEPTIONS( warn, "error removing ${key}", ("key",k) ); } - - private: - class key_compare : public leveldb::Comparator - { - public: - int Compare( const leveldb::Slice& a, const leveldb::Slice& b )const - { - FC_ASSERT( (a.size() == sizeof(Key)) && (b.size() == sizeof( Key )) ); - Key* ak = (Key*)a.data(); - Key* bk = (Key*)b.data(); - if( *ak < *bk ) return -1; - if( *ak == *bk ) return 0; - return 1; - } - - const char* Name()const { return "key_compare"; } - void FindShortestSeparator( std::string*, const leveldb::Slice& )const{} - void FindShortSuccessor( std::string* )const{}; - }; - - std::unique_ptr _db; - std::unique_ptr _cache; - key_compare _comparer; - - ldb::ReadOptions _read_options; - ldb::ReadOptions _iter_options; - ldb::WriteOptions _write_options; - ldb::WriteOptions _sync_options; - }; - -} } // graphene::db diff --git a/libraries/db/include/graphene/db/object_database.hpp b/libraries/db/include/graphene/db/object_database.hpp index 2a27bbdb..e070125e 100644 --- a/libraries/db/include/graphene/db/object_database.hpp +++ b/libraries/db/include/graphene/db/object_database.hpp @@ -20,9 +20,6 @@ #include #include -#include -#include - #include #include @@ -150,7 +147,6 @@ namespace graphene { namespace db { fc::path _data_dir; vector< vector< unique_ptr > > _index; - shared_ptr >> _object_id_to_object; }; } } // graphene::db diff --git a/libraries/db/object_database.cpp b/libraries/db/object_database.cpp index d8627e53..9b649c81 100644 --- a/libraries/db/object_database.cpp +++ b/libraries/db/object_database.cpp @@ -28,19 +28,12 @@ object_database::object_database() { _index.resize(255); _undo_db.enable(); - - _object_id_to_object = std::make_shared>>(); } object_database::~object_database(){} void object_database::close() { - if( _object_id_to_object->is_open() ) - { - flush(); - _object_id_to_object->close(); - } } const object* object_database::find_object( object_id_type id )const @@ -72,20 +65,14 @@ index& object_database::get_mutable_index(uint8_t space_id, uint8_t type_id) void object_database::flush() { - if( !_object_id_to_object->is_open() ) - return; - - vector next_ids; - for( auto& space : _index ) - for( const unique_ptr& type_index : space ) - if( type_index ) - { - type_index->inspect_all_objects([&] (const object& obj) { - _object_id_to_object->store(obj.id, obj.pack()); - }); - next_ids.push_back( type_index->get_next_id() ); - } - _object_id_to_object->store( object_id_type(), fc::raw::pack(next_ids) ); + ilog("Save object_database in ${d}", ("d", _data_dir)); + for( uint32_t space = 0; space < _index.size(); ++space ) + { + const auto types = _index[space].size(); + for( uint32_t type = 0; type < types; ++type ) + if( _index[space][type] ) + _index[space][type]->save( _data_dir / "object_database" / fc::to_string(space)/fc::to_string(type) ); + } } void object_database::wipe(const fc::path& data_dir) @@ -99,35 +86,12 @@ void object_database::wipe(const fc::path& data_dir) void object_database::open( const fc::path& data_dir ) { try { ilog("Open object_database in ${d}", ("d", data_dir)); - - _object_id_to_object->open( data_dir / "object_database" / "objects" ); - - for( auto& space : _index ) - { - for( auto& type_index : space ) - { - if( type_index ) - { - type_index->open( _object_id_to_object ); - } - } - } - try { - auto next_ids = fc::raw::unpack>( _object_id_to_object->fetch( object_id_type() ) ); - wdump((next_ids)); - for( auto id : next_ids ) - { - try { - get_mutable_index( id ).set_next_id( id ); - } FC_CAPTURE_AND_RETHROW( (id) ); - } - } - catch ( const fc::exception& e ) - { - // dlog( "unable to fetch next ids, must be new object_database\n ${e}", ("e",e.to_detail_string()) ); - } - _data_dir = data_dir; + for( uint32_t space = 0; space < _index.size(); ++space ) + for( uint32_t type = 0; type < _index[space].size(); ++type ) + if( _index[space][type] ) + _index[space][type]->open( _data_dir / "object_database" / fc::to_string(space)/fc::to_string(type) ); + } FC_CAPTURE_AND_RETHROW( (data_dir) ) } diff --git a/libraries/db/upgrade_leveldb.cpp b/libraries/db/upgrade_leveldb.cpp deleted file mode 100644 index e3e326a9..00000000 --- a/libraries/db/upgrade_leveldb.cpp +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (c) 2015, Cryptonomex, Inc. - * All rights reserved. - * - * This source code is provided for evaluation in private test networks only, until September 8, 2015. After this date, this license expires and - * the code may not be used, modified or distributed for any purpose. Redistribution and use in source and binary forms, with or without modification, - * are permitted until September 8, 2015, provided that the following conditions are met: - * - * 1. The code and/or derivative works are used only for private test networks consisting of no more than 10 P2P nodes. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace graphene { namespace db { - - upgrade_db_mapper& upgrade_db_mapper::instance() - { - static upgrade_db_mapper mapper; - return mapper; - } - - int32_t upgrade_db_mapper::add_type( const std::string& type_name, const upgrade_db_function& function) - { - _upgrade_db_function_registry[type_name] = function; - return 0; - } - - - // this code has no graphene dependencies, and it - // could be moved to fc, if fc ever adds a leveldb dependency - void try_upgrade_db( const fc::path& dir, leveldb::DB* dbase, const char* record_type, size_t record_type_size ) - { - size_t old_record_type_size = 0; - std::string old_record_type; - fc::path record_type_filename = dir / "RECORD_TYPE"; - //if no RECORD_TYPE file exists - if ( !boost::filesystem::exists( record_type_filename ) ) - { - //must be original type for the database - old_record_type = record_type; - int last_char = old_record_type.length() - 1; - //strip version number from current_record_name and append 0 to set old_record_type (e.g. mytype0) - while (last_char >= 0 && isdigit(old_record_type[last_char])) - { - --last_char; - } - - //upgradeable record types should always end with version number - if( 'v' != old_record_type[last_char] ) - { - //ilog("Database ${db} is not upgradeable",("db",dir.to_native_ansi_path())); - return; - } - - ++last_char; - old_record_type[last_char] = '0'; - old_record_type.resize(last_char+1); - } - else //read record type from file - { - boost::filesystem::ifstream is(record_type_filename); - char buffer[120]; - is.getline(buffer,120); - old_record_type = buffer; - is >> old_record_type_size; - } - if (old_record_type != record_type) - { - //check if upgrade function in registry - auto upgrade_function_itr = upgrade_db_mapper::instance()._upgrade_db_function_registry.find( old_record_type ); - if (upgrade_function_itr != upgrade_db_mapper::instance()._upgrade_db_function_registry.end()) - { - ilog("Upgrading database ${db} from ${old} to ${new}",("db",dir.preferred_string()) - ("old",old_record_type) - ("new",record_type)); - //update database's RECORD_TYPE to new record type name - boost::filesystem::ofstream os(record_type_filename); - os << record_type << std::endl; - os << record_type_size; - //upgrade the database using upgrade function - upgrade_function_itr->second(dbase); - } - else - { - elog("In ${db}, record types ${old} and ${new} do not match, but no upgrade function found!", - ("db",dir.preferred_string())("old",old_record_type)("new",record_type)); - } - } - else if (old_record_type_size == 0) //if record type file never created, create it now - { - boost::filesystem::ofstream os(record_type_filename); - os << record_type << std::endl; - os << record_type_size; - } - else if (old_record_type_size != record_type_size) - { - elog("In ${db}, record type matches ${new}, but record sizes do not match!", - ("db",dir.preferred_string())("new",record_type)); - - } - } -} } // namespace graphene::db diff --git a/libraries/leveldb b/libraries/leveldb deleted file mode 160000 index 7d41e6f8..00000000 --- a/libraries/leveldb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7d41e6f89ff04ce9e6a742932924796f69c6e23d diff --git a/libraries/net/CMakeLists.txt b/libraries/net/CMakeLists.txt index b85b581e..7cce05cf 100644 --- a/libraries/net/CMakeLists.txt +++ b/libraries/net/CMakeLists.txt @@ -10,7 +10,7 @@ set(SOURCES node.cpp add_library( graphene_net ${SOURCES} ${HEADERS} ) target_link_libraries( graphene_net - PUBLIC fc graphene_db leveldb ) + PUBLIC fc graphene_db ) target_include_directories( graphene_net PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" PRIVATE "${CMAKE_SOURCE_DIR}/libraries/chain/include" diff --git a/libraries/net/peer_database.cpp b/libraries/net/peer_database.cpp index 204ec638..4ba0273b 100644 --- a/libraries/net/peer_database.cpp +++ b/libraries/net/peer_database.cpp @@ -28,7 +28,7 @@ #include #include -#include +//#include @@ -76,8 +76,8 @@ namespace graphene { namespace net { > > potential_peer_set; //private: - typedef graphene::db::level_pod_map potential_peer_leveldb; - potential_peer_leveldb _leveldb; + //typedef graphene::db::level_pod_map potential_peer_leveldb; + //potential_peer_leveldb _leveldb; potential_peer_set _potential_peer_set; @@ -109,6 +109,7 @@ namespace graphene { namespace net { void peer_database_impl::open(const fc::path& databaseFilename) { + /* try { _leveldb.open(databaseFilename); @@ -135,16 +136,18 @@ namespace graphene { namespace net { iter = _potential_peer_set.erase(iter); } } + */ } void peer_database_impl::close() { - _leveldb.close(); + //_leveldb.close(); _potential_peer_set.clear(); } void peer_database_impl::clear() { + /* auto iter = _leveldb.begin(); while (iter.valid()) { @@ -159,6 +162,7 @@ namespace graphene { namespace net { // shouldn't happen, and if it does there's not much we can do } } + */ _potential_peer_set.clear(); } @@ -167,7 +171,7 @@ namespace graphene { namespace net { auto iter = _potential_peer_set.get().find(endpointToErase); if (iter != _potential_peer_set.get().end()) { - _leveldb.remove(iter->database_key); + //_leveldb.remove(iter->database_key); _potential_peer_set.get().erase(iter); } } @@ -178,16 +182,16 @@ namespace graphene { namespace net { if (iter != _potential_peer_set.get().end()) { _potential_peer_set.get().modify(iter, [&updatedRecord](potential_peer_database_entry& entry) { entry.peer_record = updatedRecord; }); - _leveldb.store(iter->database_key, updatedRecord); + //_leveldb.store(iter->database_key, updatedRecord); } else { uint32_t last_database_key; - _leveldb.last(last_database_key); + //_leveldb.last(last_database_key); uint32_t new_database_key = last_database_key + 1; potential_peer_database_entry new_database_entry(new_database_key, updatedRecord); _potential_peer_set.get().insert(new_database_entry); - _leveldb.store(new_database_key, updatedRecord); + //_leveldb.store(new_database_key, updatedRecord); } } @@ -314,12 +318,14 @@ namespace graphene { namespace net { std::vector peer_database::get_all()const { std::vector results; + /* auto itr = my->_leveldb.begin(); while( itr.valid() ) { results.push_back( itr.value() ); ++itr; } + */ return results; } diff --git a/tests/common/database_fixture.hpp b/tests/common/database_fixture.hpp index 7010a29f..f17e7cd2 100644 --- a/tests/common/database_fixture.hpp +++ b/tests/common/database_fixture.hpp @@ -20,6 +20,7 @@ #include #include #include +#include using namespace graphene::db; From 109d95cbda947be90616d622ab718184b329f5ff Mon Sep 17 00:00:00 2001 From: Vikram Rajkumar Date: Tue, 16 Jun 2015 17:02:13 -0400 Subject: [PATCH 6/7] Fix build --- tests/intense/block_tests.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/intense/block_tests.cpp b/tests/intense/block_tests.cpp index 993cf074..1a971b29 100644 --- a/tests/intense/block_tests.cpp +++ b/tests/intense/block_tests.cpp @@ -166,7 +166,7 @@ BOOST_FIXTURE_TEST_CASE( update_account_keys, database_fixture ) // size() < num_active_keys is possible when some keys are duplicates create_op.active.weight_threshold = create_op.active.auths.size(); - create_op.memo_key = key_ids[ *(it++) ] ; + create_op.options.memo_key = key_ids[ *(it++) ] ; create_op.registrar = sam_account_object.id; trx.operations.push_back( create_op ); // trx.sign( sam_key ); @@ -191,6 +191,7 @@ BOOST_FIXTURE_TEST_CASE( update_account_keys, database_fixture ) update_op.account = alice_account_id; update_op.owner = authority(); update_op.active = authority(); + update_op.new_options = create_op.options; for( int owner_index=0; owner_indexauths[ key_ids[ *(it++) ] ] = 1; @@ -200,7 +201,8 @@ BOOST_FIXTURE_TEST_CASE( update_account_keys, database_fixture ) update_op.active->auths[ key_ids[ *(it++) ] ] = 1; // size() < num_active_keys is possible when some keys are duplicates update_op.active->weight_threshold = update_op.active->auths.size(); - update_op.memo_key = key_ids[ *(it++) ] ; + FC_ASSERT( update_op.new_options.valid() ); + update_op.new_options->memo_key = key_ids[ *(it++) ] ; trx.operations.push_back( update_op ); for( int i=0; i Date: Tue, 16 Jun 2015 18:45:33 -0400 Subject: [PATCH 7/7] tests pass again --- libraries/chain/block.cpp | 1 + libraries/chain/block_database.cpp | 71 ++++++++++++------ libraries/chain/db_block.cpp | 6 +- libraries/chain/db_management.cpp | 4 ++ libraries/db/include/graphene/db/index.hpp | 6 +- libraries/db/object_database.cpp | 1 + tests/tests/block_tests.cpp | 84 +++++++++++++++++++--- 7 files changed, 139 insertions(+), 34 deletions(-) diff --git a/libraries/chain/block.cpp b/libraries/chain/block.cpp index ad731b84..47ab8d37 100644 --- a/libraries/chain/block.cpp +++ b/libraries/chain/block.cpp @@ -53,6 +53,7 @@ namespace graphene { namespace chain { checksum_type signed_block::calculate_merkle_root()const { if( transactions.size() == 0 ) return checksum_type(); + wdump((transactions.size())); vector ids; ids.resize( ((transactions.size() + 1)/2)*2 ); diff --git a/libraries/chain/block_database.cpp b/libraries/chain/block_database.cpp index 94389571..16cf76db 100644 --- a/libraries/chain/block_database.cpp +++ b/libraries/chain/block_database.cpp @@ -8,12 +8,29 @@ struct index_entry uint32_t block_size = 0; block_id_type block_id; }; + }} +FC_REFLECT( graphene::chain::index_entry, (block_pos)(block_size)(block_id) ); + +namespace graphene { namespace chain { void block_database::open( const fc::path& dbdir ) -{ - _block_num_to_pos.open( (dbdir/"index").generic_string().c_str(), std::fstream::binary ); - _blocks.open( (dbdir/"blocks").generic_string().c_str(), std::fstream::binary ); -} +{ try { + idump((sizeof(index_entry)) ); + fc::create_directories(dbdir); + _block_num_to_pos.exceptions(std::ios_base::failbit | std::ios_base::badbit); + _blocks.exceptions(std::ios_base::failbit | std::ios_base::badbit); + + if( !fc::exists( dbdir/"index" ) ) + { + _block_num_to_pos.open( (dbdir/"index").generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out | std::fstream::trunc); + _blocks.open( (dbdir/"blocks").generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out | std::fstream::trunc); + } + else + { + _block_num_to_pos.open( (dbdir/"index").generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out ); + _blocks.open( (dbdir/"blocks").generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out ); + } +} FC_CAPTURE_AND_RETHROW( (dbdir) ) } bool block_database::is_open()const @@ -51,7 +68,7 @@ void block_database::store( const block_id_type& id, const signed_block& b ) void block_database::remove( const block_id_type& id ) -{ +{ try { index_entry e; auto index_pos = sizeof(e)*block_header::num_from_id(id); _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); @@ -60,12 +77,13 @@ void block_database::remove( const block_id_type& id ) _block_num_to_pos.seekg( index_pos ); _block_num_to_pos.read( (char*)&e, sizeof(e) ); - FC_ASSERT( e.block_id == id ); - - e.block_size = 0; - _block_num_to_pos.seekp( sizeof(e)*block_header::num_from_id(id) ); - _block_num_to_pos.write( (char*)&e, sizeof(e) ); -} + if( e.block_id == id ) + { + e.block_size = 0; + _block_num_to_pos.seekp( sizeof(e)*block_header::num_from_id(id) ); + _block_num_to_pos.write( (char*)&e, sizeof(e) ); + } +} FC_CAPTURE_AND_RETHROW( (id) ) } @@ -100,7 +118,7 @@ block_id_type block_database::fetch_block_id( uint32_t block_num )const optional block_database::fetch_optional( const block_id_type& id )const -{ +{ try { index_entry e; auto index_pos = sizeof(e)*block_header::num_from_id(id); _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); @@ -114,25 +132,31 @@ optional block_database::fetch_optional( const block_id_type& id ) vector data( e.block_size ); _blocks.seekg( e.block_pos ); _blocks.read( data.data(), e.block_size ); - return fc::raw::unpack(data); -} + auto result = fc::raw::unpack(data); + FC_ASSERT( result.id() == e.block_id ); + return result; +} FC_CAPTURE_AND_RETHROW( (id) ) } optional block_database::fetch_by_number( uint32_t block_num )const -{ +{ try { index_entry e; auto index_pos = sizeof(e)*block_num; _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); FC_ASSERT( _block_num_to_pos.tellg() > index_pos ); - _block_num_to_pos.seekg( index_pos ); + _block_num_to_pos.seekg( index_pos, _block_num_to_pos.beg ); + wdump((int64_t(_block_num_to_pos.tellg())) ); _block_num_to_pos.read( (char*)&e, sizeof(e) ); + wdump((block_num)(e)); vector data( e.block_size ); _blocks.seekg( e.block_pos ); _blocks.read( data.data(), e.block_size ); - return fc::raw::unpack(data); -} + auto result = fc::raw::unpack(data); + FC_ASSERT( result.id() == e.block_id ); + return result; +} FC_CAPTURE_AND_RETHROW( (block_num) ) } optional block_database::last()const @@ -145,14 +169,21 @@ optional block_database::last()const _block_num_to_pos.seekg( -sizeof(index_entry), _block_num_to_pos.end ); _block_num_to_pos.read( (char*)&e, sizeof(e) ); - + while( e.block_size == 0 && _blocks.tellg() > 0 ) + { + _block_num_to_pos.seekg( -sizeof(index_entry), _block_num_to_pos.cur ); + _block_num_to_pos.read( (char*)&e, sizeof(e) ); + } + if( e.block_size == 0 ) return optional(); vector data( e.block_size ); _blocks.seekg( e.block_pos ); _blocks.read( data.data(), e.block_size ); - return fc::raw::unpack(data); + auto result = fc::raw::unpack(data); + wdump((result)); + return result; } diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp index e8493820..d82dde79 100644 --- a/libraries/chain/db_block.cpp +++ b/libraries/chain/db_block.cpp @@ -84,7 +84,6 @@ bool database::push_block( const signed_block& new_block, uint32_t skip ) { try { if( !(skip&skip_fork_db) ) { - wdump((new_block.id())(new_block.previous)); auto new_head = _fork_db.push_block( new_block ); //If the head block from the longest chain does not build off of the current head, we need to switch forks. if( new_head->data.previous != head_block_id() ) @@ -115,12 +114,13 @@ bool database::push_block( const signed_block& new_block, uint32_t skip ) try { auto session = _undo_db.start_undo_session(); apply_block( (*ritr)->data, skip ); - _block_id_to_block.store( new_block.id(), (*ritr)->data ); + _block_id_to_block.store( (*ritr)->id, (*ritr)->data ); session.commit(); } catch ( const fc::exception& e ) { except = e; } if( except ) { + wdump((except->to_detail_string())); elog( "Encountered error when switching to a longer fork at id ${id}. Going back.", ("id", (*ritr)->id) ); // remove the rest of branches.first from the fork_db, those blocks are invalid @@ -325,7 +325,7 @@ void database::apply_block( const signed_block& next_block, uint32_t skip ) { try { _applied_ops.clear(); - FC_ASSERT( (skip & skip_merkle_check) || next_block.transaction_merkle_root == next_block.calculate_merkle_root() ); + FC_ASSERT( (skip & skip_merkle_check) || next_block.transaction_merkle_root == next_block.calculate_merkle_root(), "", ("next_block.transaction_merkle_root",next_block.transaction_merkle_root)("calc",next_block.calculate_merkle_root())("next_block",next_block)("id",next_block.id()) ); const witness_object& signing_witness = validate_block_header(skip, next_block); const auto& global_props = get_global_properties(); diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp index 5651fe1e..5ab3c837 100644 --- a/libraries/chain/db_management.cpp +++ b/libraries/chain/db_management.cpp @@ -41,7 +41,10 @@ void database::open( const fc::path& data_dir, const genesis_allocation& initial _block_id_to_block.open( data_dir / "database" / "block_num_to_block" ); if( !find(global_property_id_type()) ) + { + ilog( "Init Genesis State" ); init_genesis(initial_allocation); + } _pending_block.previous = head_block_id(); _pending_block.timestamp = head_block_time(); @@ -96,6 +99,7 @@ void database::close(uint32_t blocks_to_rewind) for(uint32_t i = 0; i < blocks_to_rewind && head_block_num() > 0; ++i) pop_block(); + object_database::flush(); object_database::close(); if( _block_id_to_block.is_open() ) diff --git a/libraries/db/include/graphene/db/index.hpp b/libraries/db/include/graphene/db/index.hpp index af95414d..84c4103a 100644 --- a/libraries/db/include/graphene/db/index.hpp +++ b/libraries/db/include/graphene/db/index.hpp @@ -177,7 +177,7 @@ namespace graphene { namespace db { virtual void set_next_id( object_id_type id )override { _next_id = id; } virtual void open( const path& db )override - { + { if( !fc::exists( db ) ) return; fc::file_mapping fm( db.generic_string().c_str(), fc::read_only ); fc::mapped_region mr( fm, fc::read_only, 0, fc::file_size(db) ); @@ -195,7 +195,9 @@ namespace graphene { namespace db { virtual void save( const path& db ) override { - std::ofstream out( db.generic_string(), std::ofstream::binary ); + std::ofstream out( db.generic_string(), + std::ofstream::binary | std::ofstream::out | std::ofstream::trunc ); + FC_ASSERT( out ); out.write( (char*)&_next_id, sizeof(_next_id) ); this->inspect_all_objects( [&]( const object& o ) { auto vec = fc::raw::pack( static_cast(o) ); diff --git a/libraries/db/object_database.cpp b/libraries/db/object_database.cpp index 9b649c81..dc5cba58 100644 --- a/libraries/db/object_database.cpp +++ b/libraries/db/object_database.cpp @@ -68,6 +68,7 @@ void object_database::flush() ilog("Save object_database in ${d}", ("d", _data_dir)); for( uint32_t space = 0; space < _index.size(); ++space ) { + fc::create_directories( _data_dir / "object_database" / fc::to_string(space) ); const auto types = _index[space].size(); for( uint32_t type = 0; type < types; ++type ) if( _index[space][type] ) diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp index c74e182d..3ff4ea90 100644 --- a/tests/tests/block_tests.cpp +++ b/tests/tests/block_tests.cpp @@ -37,6 +37,73 @@ using namespace graphene::chain; BOOST_AUTO_TEST_SUITE(block_tests) +BOOST_AUTO_TEST_CASE( block_database_test ) +{ + try { + fc::temp_directory data_dir; + + block_database bdb; + bdb.open( data_dir.path() ); + FC_ASSERT( bdb.is_open() ); + bdb.close(); + FC_ASSERT( !bdb.is_open() ); + bdb.open( data_dir.path() ); + + signed_block b; + for( uint32_t i = 0; i < 5; ++i ) + { + if( i > 0 ) b.previous = b.id(); + b.witness = witness_id_type(i+1); + edump((b)); + bdb.store( b.id(), b ); + + auto fetch = bdb.fetch_by_number( b.block_num() ); + idump((fetch)); + FC_ASSERT( fetch.valid() ); + FC_ASSERT( fetch->witness == b.witness ); + fetch = bdb.fetch_by_number( i+1 ); + idump((fetch)); + FC_ASSERT( fetch.valid() ); + FC_ASSERT( fetch->witness == b.witness ); + fetch = bdb.fetch_optional( b.id() ); + idump((fetch)); + FC_ASSERT( fetch.valid() ); + FC_ASSERT( fetch->witness == b.witness ); + } + ilog("-----------" ); + + for( uint32_t i = 1; i < 5; ++i ) + { + auto blk = bdb.fetch_by_number( i ); + FC_ASSERT( blk.valid() ); + idump((blk)(i)); + FC_ASSERT( blk->witness == witness_id_type(blk->block_num()) ); + } + + auto last = bdb.last(); + FC_ASSERT( last ); + FC_ASSERT( last->id() == b.id() ); + + bdb.close(); + bdb.open( data_dir.path() ); + last = bdb.last(); + FC_ASSERT( last ); + FC_ASSERT( last->id() == b.id() ); + + for( uint32_t i = 0; i < 5; ++i ) + { + auto blk = bdb.fetch_by_number( i+1 ); + FC_ASSERT( blk.valid() ); + idump((blk)(i)); + FC_ASSERT( blk->witness == witness_id_type(blk->block_num()) ); + } + + } catch (fc::exception& e) { + edump((e.to_detail_string())); + throw; + } +} + BOOST_AUTO_TEST_CASE( generate_empty_blocks ) { try { @@ -142,7 +209,7 @@ BOOST_AUTO_TEST_CASE( fork_blocks ) db2.open( data_dir2.path(), genesis_allocation() ); auto delegate_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("genesis")) ); - for( uint32_t i = 0; i < 20; ++i ) + for( uint32_t i = 0; i < 10; ++i ) { now += db1.block_interval(); auto b = db1.generate_block( now, db1.get_scheduled_witness( 1 ).first, delegate_priv_key ); @@ -150,20 +217,19 @@ BOOST_AUTO_TEST_CASE( fork_blocks ) db2.push_block(b); } FC_CAPTURE_AND_RETHROW( ("db2") ); } - for( uint32_t i = 20; i < 23; ++i ) + for( uint32_t i = 10; i < 13; ++i ) { now += db1.block_interval(); auto b = db1.generate_block( now, db1.get_scheduled_witness( 1 ).first, delegate_priv_key ); } string db1_tip = db1.head_block_id().str(); - for( uint32_t i = 23; i < 26; ++i ) + for( uint32_t i = 13; i < 16; ++i ) { now += db2.block_interval(); auto b = db2.generate_block( now, db2.get_scheduled_witness( db2.get_slot_at_time( now ) ).first, delegate_priv_key ); // notify both databases of the new block. // only db2 should switch to the new fork, db1 should not db1.push_block(b); - db2.push_block(b); BOOST_CHECK_EQUAL(db1.head_block_id().str(), db1_tip); BOOST_CHECK_EQUAL(db2.head_block_id().str(), b.id().str()); } @@ -171,8 +237,8 @@ BOOST_AUTO_TEST_CASE( fork_blocks ) //The two databases are on distinct forks now, but at the same height. Make a block on db2, make it invalid, then //pass it to db1 and assert that db1 doesn't switch to the new fork. signed_block good_block; - BOOST_CHECK_EQUAL(db1.head_block_num(), 23); - BOOST_CHECK_EQUAL(db2.head_block_num(), 23); + BOOST_CHECK_EQUAL(db1.head_block_num(), 13); + BOOST_CHECK_EQUAL(db2.head_block_num(), 13); { now += db2.block_interval(); auto b = db2.generate_block( now, db2.get_scheduled_witness( 1 ).first, delegate_priv_key ); @@ -180,14 +246,14 @@ BOOST_AUTO_TEST_CASE( fork_blocks ) b.transactions.emplace_back(signed_transaction()); b.transactions.back().operations.emplace_back(transfer_operation()); b.sign(delegate_priv_key); - BOOST_CHECK_EQUAL(b.block_num(), 24); + BOOST_CHECK_EQUAL(b.block_num(), 14); BOOST_CHECK_THROW(db1.push_block(b), fc::exception); } - BOOST_CHECK_EQUAL(db1.head_block_num(), 23); + BOOST_CHECK_EQUAL(db1.head_block_num(), 13); BOOST_CHECK_EQUAL(db1.head_block_id().str(), db1_tip); // assert that db1 switches to new fork with good block - BOOST_CHECK_EQUAL(db2.head_block_num(), 24); + BOOST_CHECK_EQUAL(db2.head_block_num(), 14); db1.push_block(good_block); BOOST_CHECK_EQUAL(db1.head_block_id().str(), db2.head_block_id().str()); } catch (fc::exception& e) {