Peerplays_Blockchain/peerplays_migrated
1
0
Fork 0
Code Issues 35 Pull requests 4 Projects Releases 20 Packages Wiki Activity Actions

Merge pull request #297 from abitmore/292-acc-his-prune

Browse source 
Account history: option to prune old data (#292)
This commit is contained in:
Abit 2017-06-06 00:42:54 +02:00 committed by GitHub
commit feabafd45f
4 changed files with 129 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
libraries/app/api.cpp
View file

@ -493,13 +493,14 @@ namespace graphene { namespace app {
const auto& db = *_app.chain_database();
FC_ASSERT(limit <= 100);
vector<operation_history_object> result;
const auto& stats = account(db).statistics(db);
if( start == 0 )
start = account(db).statistics(db).total_ops;
start = stats.total_ops;
else
start = min( account(db).statistics(db).total_ops, start );
start = min( stats.total_ops, start );
if( start >= stop && start > 0 && limit > 0 )
if( start >= stop && start > stats.removed_ops && limit > 0 )
{
const auto& hist_idx = db.get_index_type<account_transaction_history_index>();
const auto& by_seq_idx = hist_idx.indices().get<by_seq>();

5
libraries/chain/include/graphene/chain/account_object.hpp
View file

@ -50,7 +50,10 @@ namespace graphene { namespace chain {
* Keep the most recent operation as a root pointer to a linked list of the transaction history.
*/
account_transaction_history_id_type most_recent_op;
/** Total operations related to this account. */
uint32_t total_ops = 0;
/** Total operations related to this account that has been removed from the database. */
uint32_t removed_ops = 0;
/**
* When calculating votes it is necessary to know how much is stored in orders (and thus unavailable for
@ -386,7 +389,7 @@ FC_REFLECT_DERIVED( graphene::chain::account_statistics_object,
(graphene::chain::object),
(owner)
(most_recent_op)
(total_ops)
(total_ops)(removed_ops)
(total_core_in_orders)
(lifetime_fees_paid)
(pending_fees)(pending_vested_fees)

4
libraries/chain/include/graphene/chain/operation_history_object.hpp
View file

@ -102,6 +102,7 @@ namespace graphene { namespace chain {
struct by_id;
struct by_seq;
struct by_op;
struct by_opid;
typedef multi_index_container<
account_transaction_history_object,
indexed_by<
@ -117,6 +118,9 @@ typedef multi_index_container<
member< account_transaction_history_object, account_id_type, &account_transaction_history_object::account>,
member< account_transaction_history_object, operation_history_id_type, &account_transaction_history_object::operation_id>
>
>,
ordered_non_unique< tag<by_opid>,
member< account_transaction_history_object, operation_history_id_type, &account_transaction_history_object::operation_id>
>
>
> account_transaction_history_multi_index_type;

174
libraries/plugins/account_history/account_history_plugin.cpp
View file

@ -66,6 +66,11 @@ class account_history_plugin_impl
flat_set<account_id_type> _tracked_accounts;
bool _partial_operations = false;
primary_index< simple_index< operation_history_object > >* _oho_index;
uint32_t _max_ops_per_account = -1;
private:
/** add one history record, then check and remove the earliest history record */
void add_account_history( const account_id_type account_id, const operation_history_id_type op_id );
};
account_history_plugin_impl::~account_history_plugin_impl()
@ -89,39 +94,26 @@ void account_history_plugin_impl::update_account_histories( const signed_block&
} ) );
};
if (_partial_operations)
if( !o_op.valid() || ( _max_ops_per_account == 0 && _partial_operations ) )
{
if( !o_op.valid() )
{
_oho_index->use_next_id();
continue;
}
// Note: the 2nd and 3rd checks above are for better performance, when the db is not clean,
// they will break consistency of account_stats.total_ops and removed_ops and most_recent_op
_oho_index->use_next_id();
continue;
}
else
{
else if( !_partial_operations )
// add to the operation history index
oho = create_oho();
if( !o_op.valid() )
{
ilog( "removing failed operation with ID: ${id}", ("id", oho->id) );
db.remove( *oho );
continue;
}
}
const operation_history_object& op = *o_op;
// get the set of accounts this operation applies to
flat_set<account_id_type> impacted;
vector<authority> other;
operation_get_required_authorities( op.op, impacted, impacted, other );
operation_get_required_authorities( op.op, impacted, impacted, other ); // fee_payer is added here
if( op.op.which() == operation::tag< account_create_operation >::value )
{
if (!oho.valid()) { oho = create_oho(); }
impacted.insert( oho->result.get<object_id_type>() );
}
impacted.insert( op.result.get<object_id_type>() );
else
graphene::app::operation_get_impacted_accounts( op.op, impacted );
@ -129,48 +121,52 @@ void account_history_plugin_impl::update_account_histories( const signed_block&
for( auto& item : a.account_auths )
impacted.insert( item.first );
// for each operation this account applies to that is in the config link it into the history
if( _tracked_accounts.size() == 0 )
{
if (!impacted.empty() && !oho.valid()) { oho = create_oho(); }
for( auto& account_id : impacted )
{
// we don't do index_account_keys here anymore, because
// that indexing now happens in observers' post_evaluate()
// be here, either _max_ops_per_account > 0, or _partial_operations == false, or both
// if _partial_operations == false, oho should have been created above
// so the only case should be checked here is:
// whether need to create oho if _max_ops_per_account > 0 and _partial_operations == true
// add history
const auto& stats_obj = account_id(db).statistics(db);
const auto& ath = db.create<account_transaction_history_object>( [&]( account_transaction_history_object& obj ){
obj.operation_id = oho->id;
obj.account = account_id;
obj.sequence = stats_obj.total_ops+1;
obj.next = stats_obj.most_recent_op;
});
db.modify( stats_obj, [&]( account_statistics_object& obj ){
obj.most_recent_op = ath.id;
obj.total_ops = ath.sequence;
});
// for each operation this account applies to that is in the config link it into the history
if( _tracked_accounts.size() == 0 ) // tracking all accounts
{
// if tracking all accounts, when impacted is not empty (although it will always be),
// still need to create oho if _max_ops_per_account > 0 and _partial_operations == true
// so always need to create oho if not done
if (!impacted.empty() && !oho.valid()) { oho = create_oho(); }
if( _max_ops_per_account > 0 )
{
// Note: the check above is for better performance, when the db is not clean,
// it breaks consistency of account_stats.total_ops and removed_ops and most_recent_op,
// but it ensures it's safe to remove old entries in add_account_history(...)
for( auto& account_id : impacted )
{
// we don't do index_account_keys here anymore, because
// that indexing now happens in observers' post_evaluate()
// add history
add_account_history( account_id, oho->id );
}
}
}
else
else // tracking a subset of accounts
{
for( auto account_id : _tracked_accounts )
// whether need to create oho if _max_ops_per_account > 0 and _partial_operations == true ?
// the answer: only need to create oho if a tracked account is impacted and need to save history
if( _max_ops_per_account > 0 )
{
if( impacted.find( account_id ) != impacted.end() )
// Note: the check above is for better performance, when the db is not clean,
// it breaks consistency of account_stats.total_ops and removed_ops and most_recent_op,
// but it ensures it's safe to remove old entries in add_account_history(...)
for( auto account_id : _tracked_accounts )
{
if (!oho.valid()) { oho = create_oho(); }
// add history
const auto& stats_obj = account_id(db).statistics(db);
const auto& ath = db.create<account_transaction_history_object>( [&]( account_transaction_history_object& obj ){
obj.operation_id = oho->id;
obj.account = account_id;
obj.sequence = stats_obj.total_ops+1;
obj.next = stats_obj.most_recent_op;
});
db.modify( stats_obj, [&]( account_statistics_object& obj ){
obj.most_recent_op = ath.id;
obj.total_ops = ath.sequence;
});
if( impacted.find( account_id ) != impacted.end() )
{
if (!oho.valid()) { oho = create_oho(); }
// add history
add_account_history( account_id, oho->id );
}
}
}
}
@ -178,6 +174,66 @@ void account_history_plugin_impl::update_account_histories( const signed_block&
_oho_index->use_next_id();
}
}
void account_history_plugin_impl::add_account_history( const account_id_type account_id, const operation_history_id_type op_id )
{
graphene::chain::database& db = database();
const auto& stats_obj = account_id(db).statistics(db);
// add new entry
const auto& ath = db.create<account_transaction_history_object>( [&]( account_transaction_history_object& obj ){
obj.operation_id = op_id;
obj.account = account_id;
obj.sequence = stats_obj.total_ops + 1;
obj.next = stats_obj.most_recent_op;
});
db.modify( stats_obj, [&]( account_statistics_object& obj ){
obj.most_recent_op = ath.id;
obj.total_ops = ath.sequence;
});
// remove the earliest account history entry if too many
// _max_ops_per_account is guaranteed to be non-zero outside
if( stats_obj.total_ops - stats_obj.removed_ops > _max_ops_per_account )
{
// look for the earliest entry
const auto& his_idx = db.get_index_type<account_transaction_history_index>();
const auto& by_seq_idx = his_idx.indices().get<by_seq>();
auto itr = by_seq_idx.lower_bound( boost::make_tuple( account_id, 0 ) );
// make sure don't remove the one just added
if( itr != by_seq_idx.end() && itr->account == account_id && itr->id != ath.id )
{
// if found, remove the entry, and adjust account stats object
const auto remove_op_id = itr->operation_id;
const auto itr_remove = itr;
++itr;
db.remove( *itr_remove );
db.modify( stats_obj, [&]( account_statistics_object& obj ){
obj.removed_ops = obj.removed_ops + 1;
});
// modify previous node's next pointer
// this should be always true, but just have a check here
if( itr != by_seq_idx.end() && itr->account == account_id )
{
db.modify( *itr, [&]( account_transaction_history_object& obj ){
obj.next = account_transaction_history_id_type();
});
}
// else need to modify the head pointer, but it shouldn't be true
// remove the operation history entry (1.11.x) if configured and no reference left
if( _partial_operations )
{
// check for references
const auto& by_opid_idx = his_idx.indices().get<by_opid>();
if( by_opid_idx.find( remove_op_id ) == by_opid_idx.end() )
{
// if no reference, remove
db.remove( remove_op_id(db) );
}
}
}
}
}
} // end namespace detail
@ -207,6 +263,7 @@ void account_history_plugin::plugin_set_program_options(
cli.add_options()
("track-account", boost::program_options::value<std::vector<std::string>>()->composing()->multitoken(), "Account ID to track history for (may specify multiple times)")
("partial-operations", boost::program_options::value<bool>(), "Keep only those operations in memory that are related to account history tracking")
("max-ops-per-account", boost::program_options::value<uint32_t>(), "Maximum number of operations per account will be kept in memory")
;
cfg.add(cli);
}
@ -221,6 +278,9 @@ void account_history_plugin::plugin_initialize(const boost::program_options::var
if (options.count("partial-operations")) {
my->_partial_operations = options["partial-operations"].as<bool>();
}
if (options.count("max-ops-per-account")) {
my->_max_ops_per_account = options["max-ops-per-account"].as<uint32_t>();
}
}
void account_history_plugin::plugin_startup()