From 504aa5dccd88c8d3a55cd6f7ee80978d0bfa3f2c Mon Sep 17 00:00:00 2001 From: abitmore Date: Fri, 2 Jun 2017 16:12:53 +0000 Subject: [PATCH] Account history: option to prune old data (#292) --- libraries/app/api.cpp | 7 +- .../include/graphene/chain/account_object.hpp | 5 +- .../chain/operation_history_object.hpp | 4 + .../account_history_plugin.cpp | 174 ++++++++++++------ 4 files changed, 129 insertions(+), 61 deletions(-) diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index 5281c0d9..d346e121 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -493,13 +493,14 @@ namespace graphene { namespace app { const auto& db = *_app.chain_database(); FC_ASSERT(limit <= 100); vector result; + const auto& stats = account(db).statistics(db); if( start == 0 ) - start = account(db).statistics(db).total_ops; + start = stats.total_ops; else - start = min( account(db).statistics(db).total_ops, start ); + start = min( stats.total_ops, start ); - if( start >= stop && start > 0 && limit > 0 ) + if( start >= stop && start > stats.removed_ops && limit > 0 ) { const auto& hist_idx = db.get_index_type(); const auto& by_seq_idx = hist_idx.indices().get(); diff --git a/libraries/chain/include/graphene/chain/account_object.hpp b/libraries/chain/include/graphene/chain/account_object.hpp index faf59e22..522cb7bc 100644 --- a/libraries/chain/include/graphene/chain/account_object.hpp +++ b/libraries/chain/include/graphene/chain/account_object.hpp @@ -50,7 +50,10 @@ namespace graphene { namespace chain { * Keep the most recent operation as a root pointer to a linked list of the transaction history. */ account_transaction_history_id_type most_recent_op; + /** Total operations related to this account. */ uint32_t total_ops = 0; + /** Total operations related to this account that has been removed from the database. */ + uint32_t removed_ops = 0; /** * When calculating votes it is necessary to know how much is stored in orders (and thus unavailable for @@ -386,7 +389,7 @@ FC_REFLECT_DERIVED( graphene::chain::account_statistics_object, (graphene::chain::object), (owner) (most_recent_op) - (total_ops) + (total_ops)(removed_ops) (total_core_in_orders) (lifetime_fees_paid) (pending_fees)(pending_vested_fees) diff --git a/libraries/chain/include/graphene/chain/operation_history_object.hpp b/libraries/chain/include/graphene/chain/operation_history_object.hpp index ecbbc58d..eae8a01e 100644 --- a/libraries/chain/include/graphene/chain/operation_history_object.hpp +++ b/libraries/chain/include/graphene/chain/operation_history_object.hpp @@ -102,6 +102,7 @@ namespace graphene { namespace chain { struct by_id; struct by_seq; struct by_op; +struct by_opid; typedef multi_index_container< account_transaction_history_object, indexed_by< @@ -117,6 +118,9 @@ typedef multi_index_container< member< account_transaction_history_object, account_id_type, &account_transaction_history_object::account>, member< account_transaction_history_object, operation_history_id_type, &account_transaction_history_object::operation_id> > + >, + ordered_non_unique< tag, + member< account_transaction_history_object, operation_history_id_type, &account_transaction_history_object::operation_id> > > > account_transaction_history_multi_index_type; diff --git a/libraries/plugins/account_history/account_history_plugin.cpp b/libraries/plugins/account_history/account_history_plugin.cpp index a5f90ea0..f1bba84d 100644 --- a/libraries/plugins/account_history/account_history_plugin.cpp +++ b/libraries/plugins/account_history/account_history_plugin.cpp @@ -66,6 +66,11 @@ class account_history_plugin_impl flat_set _tracked_accounts; bool _partial_operations = false; primary_index< simple_index< operation_history_object > >* _oho_index; + uint32_t _max_ops_per_account = -1; + private: + /** add one history record, then check and remove the earliest history record */ + void add_account_history( const account_id_type account_id, const operation_history_id_type op_id ); + }; account_history_plugin_impl::~account_history_plugin_impl() @@ -89,39 +94,26 @@ void account_history_plugin_impl::update_account_histories( const signed_block& } ) ); }; - if (_partial_operations) + if( !o_op.valid() || ( _max_ops_per_account == 0 && _partial_operations ) ) { - if( !o_op.valid() ) - { - _oho_index->use_next_id(); - continue; - } + // Note: the 2nd and 3rd checks above are for better performance, when the db is not clean, + // they will break consistency of account_stats.total_ops and removed_ops and most_recent_op + _oho_index->use_next_id(); + continue; } - else - { + else if( !_partial_operations ) // add to the operation history index oho = create_oho(); - if( !o_op.valid() ) - { - ilog( "removing failed operation with ID: ${id}", ("id", oho->id) ); - db.remove( *oho ); - continue; - } - } - const operation_history_object& op = *o_op; // get the set of accounts this operation applies to flat_set impacted; vector other; - operation_get_required_authorities( op.op, impacted, impacted, other ); + operation_get_required_authorities( op.op, impacted, impacted, other ); // fee_payer is added here if( op.op.which() == operation::tag< account_create_operation >::value ) - { - if (!oho.valid()) { oho = create_oho(); } - impacted.insert( oho->result.get() ); - } + impacted.insert( op.result.get() ); else graphene::app::operation_get_impacted_accounts( op.op, impacted ); @@ -129,48 +121,52 @@ void account_history_plugin_impl::update_account_histories( const signed_block& for( auto& item : a.account_auths ) impacted.insert( item.first ); - // for each operation this account applies to that is in the config link it into the history - if( _tracked_accounts.size() == 0 ) - { - if (!impacted.empty() && !oho.valid()) { oho = create_oho(); } - for( auto& account_id : impacted ) - { - // we don't do index_account_keys here anymore, because - // that indexing now happens in observers' post_evaluate() + // be here, either _max_ops_per_account > 0, or _partial_operations == false, or both + // if _partial_operations == false, oho should have been created above + // so the only case should be checked here is: + // whether need to create oho if _max_ops_per_account > 0 and _partial_operations == true - // add history - const auto& stats_obj = account_id(db).statistics(db); - const auto& ath = db.create( [&]( account_transaction_history_object& obj ){ - obj.operation_id = oho->id; - obj.account = account_id; - obj.sequence = stats_obj.total_ops+1; - obj.next = stats_obj.most_recent_op; - }); - db.modify( stats_obj, [&]( account_statistics_object& obj ){ - obj.most_recent_op = ath.id; - obj.total_ops = ath.sequence; - }); + // for each operation this account applies to that is in the config link it into the history + if( _tracked_accounts.size() == 0 ) // tracking all accounts + { + // if tracking all accounts, when impacted is not empty (although it will always be), + // still need to create oho if _max_ops_per_account > 0 and _partial_operations == true + // so always need to create oho if not done + if (!impacted.empty() && !oho.valid()) { oho = create_oho(); } + + if( _max_ops_per_account > 0 ) + { + // Note: the check above is for better performance, when the db is not clean, + // it breaks consistency of account_stats.total_ops and removed_ops and most_recent_op, + // but it ensures it's safe to remove old entries in add_account_history(...) + for( auto& account_id : impacted ) + { + // we don't do index_account_keys here anymore, because + // that indexing now happens in observers' post_evaluate() + + // add history + add_account_history( account_id, oho->id ); + } } } - else + else // tracking a subset of accounts { - for( auto account_id : _tracked_accounts ) + // whether need to create oho if _max_ops_per_account > 0 and _partial_operations == true ? + // the answer: only need to create oho if a tracked account is impacted and need to save history + + if( _max_ops_per_account > 0 ) { - if( impacted.find( account_id ) != impacted.end() ) + // Note: the check above is for better performance, when the db is not clean, + // it breaks consistency of account_stats.total_ops and removed_ops and most_recent_op, + // but it ensures it's safe to remove old entries in add_account_history(...) + for( auto account_id : _tracked_accounts ) { - if (!oho.valid()) { oho = create_oho(); } - // add history - const auto& stats_obj = account_id(db).statistics(db); - const auto& ath = db.create( [&]( account_transaction_history_object& obj ){ - obj.operation_id = oho->id; - obj.account = account_id; - obj.sequence = stats_obj.total_ops+1; - obj.next = stats_obj.most_recent_op; - }); - db.modify( stats_obj, [&]( account_statistics_object& obj ){ - obj.most_recent_op = ath.id; - obj.total_ops = ath.sequence; - }); + if( impacted.find( account_id ) != impacted.end() ) + { + if (!oho.valid()) { oho = create_oho(); } + // add history + add_account_history( account_id, oho->id ); + } } } } @@ -178,6 +174,66 @@ void account_history_plugin_impl::update_account_histories( const signed_block& _oho_index->use_next_id(); } } + +void account_history_plugin_impl::add_account_history( const account_id_type account_id, const operation_history_id_type op_id ) +{ + graphene::chain::database& db = database(); + const auto& stats_obj = account_id(db).statistics(db); + // add new entry + const auto& ath = db.create( [&]( account_transaction_history_object& obj ){ + obj.operation_id = op_id; + obj.account = account_id; + obj.sequence = stats_obj.total_ops + 1; + obj.next = stats_obj.most_recent_op; + }); + db.modify( stats_obj, [&]( account_statistics_object& obj ){ + obj.most_recent_op = ath.id; + obj.total_ops = ath.sequence; + }); + // remove the earliest account history entry if too many + // _max_ops_per_account is guaranteed to be non-zero outside + if( stats_obj.total_ops - stats_obj.removed_ops > _max_ops_per_account ) + { + // look for the earliest entry + const auto& his_idx = db.get_index_type(); + const auto& by_seq_idx = his_idx.indices().get(); + auto itr = by_seq_idx.lower_bound( boost::make_tuple( account_id, 0 ) ); + // make sure don't remove the one just added + if( itr != by_seq_idx.end() && itr->account == account_id && itr->id != ath.id ) + { + // if found, remove the entry, and adjust account stats object + const auto remove_op_id = itr->operation_id; + const auto itr_remove = itr; + ++itr; + db.remove( *itr_remove ); + db.modify( stats_obj, [&]( account_statistics_object& obj ){ + obj.removed_ops = obj.removed_ops + 1; + }); + // modify previous node's next pointer + // this should be always true, but just have a check here + if( itr != by_seq_idx.end() && itr->account == account_id ) + { + db.modify( *itr, [&]( account_transaction_history_object& obj ){ + obj.next = account_transaction_history_id_type(); + }); + } + // else need to modify the head pointer, but it shouldn't be true + + // remove the operation history entry (1.11.x) if configured and no reference left + if( _partial_operations ) + { + // check for references + const auto& by_opid_idx = his_idx.indices().get(); + if( by_opid_idx.find( remove_op_id ) == by_opid_idx.end() ) + { + // if no reference, remove + db.remove( remove_op_id(db) ); + } + } + } + } +} + } // end namespace detail @@ -207,6 +263,7 @@ void account_history_plugin::plugin_set_program_options( cli.add_options() ("track-account", boost::program_options::value>()->composing()->multitoken(), "Account ID to track history for (may specify multiple times)") ("partial-operations", boost::program_options::value(), "Keep only those operations in memory that are related to account history tracking") + ("max-ops-per-account", boost::program_options::value(), "Maximum number of operations per account will be kept in memory") ; cfg.add(cli); } @@ -221,6 +278,9 @@ void account_history_plugin::plugin_initialize(const boost::program_options::var if (options.count("partial-operations")) { my->_partial_operations = options["partial-operations"].as(); } + if (options.count("max-ops-per-account")) { + my->_max_ops_per_account = options["max-ops-per-account"].as(); + } } void account_history_plugin::plugin_startup()