From 36e318a50308c2e5d02f8b1b5581ae0c74055ff7 Mon Sep 17 00:00:00 2001 From: abitmore Date: Tue, 3 Jul 2018 13:32:08 -0400 Subject: [PATCH] Improve update_expired_feeds performance #1093 --- libraries/chain/asset_evaluator.cpp | 26 ++++- libraries/chain/asset_object.cpp | 5 + libraries/chain/db_block.cpp | 5 +- libraries/chain/db_market.cpp | 21 +++- libraries/chain/db_update.cpp | 104 +++++++++++++----- .../include/graphene/chain/asset_object.hpp | 46 +++++++- .../chain/include/graphene/chain/database.hpp | 9 +- 7 files changed, 170 insertions(+), 46 deletions(-) diff --git a/libraries/chain/asset_evaluator.cpp b/libraries/chain/asset_evaluator.cpp index e51ccc1a..21378ce1 100644 --- a/libraries/chain/asset_evaluator.cpp +++ b/libraries/chain/asset_evaluator.cpp @@ -149,15 +149,16 @@ object_id_type asset_create_evaluator::do_apply( const asset_create_operation& o }); } + auto next_asset_id = db().get_index_type().get_next_id(); + asset_bitasset_data_id_type bit_asset_id; if( op.bitasset_opts.valid() ) bit_asset_id = db().create( [&]( asset_bitasset_data_object& a ) { a.options = *op.bitasset_opts; a.is_prediction_market = op.is_prediction_market; + a.asset_id = next_asset_id; }).id; - auto next_asset_id = db().get_index_type().get_next_id(); - const asset_object& new_asset = db().create( [&]( asset_object& a ) { a.issuer = op.issuer; @@ -297,17 +298,18 @@ object_id_type lottery_asset_create_evaluator::do_apply( const lottery_asset_cre }); } + auto next_asset_id = db().get_index_type().get_next_id(); + asset_bitasset_data_id_type bit_asset_id; if( op.bitasset_opts.valid() ) bit_asset_id = db().create( [&op,next_asset_id]( asset_bitasset_data_object& a ) { a.options = *op.bitasset_opts; a.is_prediction_market = op.is_prediction_market; + a.asset_id = next_asset_id; }).id; - auto next_asset_id = db().get_index_type().get_next_id(); - const asset_object& new_asset = - db().create( [&op,next_asset_id,&dyn_asset,bit_asset_id]( asset_object& a ) { + db().create( [&op,next_asset_id,&dyn_asset,bit_asset_id,this]( asset_object& a ) { a.issuer = op.issuer; a.symbol = op.symbol; a.precision = op.precision; @@ -483,6 +485,20 @@ void_result asset_update_evaluator::do_apply(const asset_update_operation& o) d.cancel_order(*itr); } + // For market-issued assets, if core change rate changed, update flag in bitasset data + if( asset_to_update->is_market_issued() + && asset_to_update->options.core_exchange_rate != o.new_options.core_exchange_rate ) + { + const auto& bitasset = asset_to_update->bitasset_data(d); + if( !bitasset.asset_cer_updated ) + { + d.modify( bitasset, [](asset_bitasset_data_object& b) + { + b.asset_cer_updated = true; + }); + } + } + d.modify(*asset_to_update, [&o](asset_object& a) { if( o.new_issuer ) a.issuer = *o.new_issuer; diff --git a/libraries/chain/asset_object.cpp b/libraries/chain/asset_object.cpp index 63df70a3..ea387932 100644 --- a/libraries/chain/asset_object.cpp +++ b/libraries/chain/asset_object.cpp @@ -61,12 +61,15 @@ void asset_bitasset_data_object::update_median_feeds(time_point_sec current_time if( current_feeds.size() < options.minimum_feeds ) { //... don't calculate a median, and set a null feed + feed_cer_updated = false; // new median cer is null, won't update asset_object anyway, set to false for better performance current_feed_publication_time = current_time; current_feed = price_feed(); return; } if( current_feeds.size() == 1 ) { + if( current_feed.core_exchange_rate != current_feeds.front().get().core_exchange_rate ) + feed_cer_updated = true; current_feed = std::move(current_feeds.front()); return; } @@ -85,6 +88,8 @@ void asset_bitasset_data_object::update_median_feeds(time_point_sec current_time #undef CALCULATE_MEDIAN_VALUE // *** End Median Calculations *** + if( current_feed.core_exchange_rate != median_feed.core_exchange_rate ) + feed_cer_updated = true; current_feed = median_feed; } diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp index 1ad84fa0..45d75fa5 100644 --- a/libraries/chain/db_block.cpp +++ b/libraries/chain/db_block.cpp @@ -592,7 +592,7 @@ void database::_apply_block( const signed_block& next_block ) const witness_object& signing_witness = validate_block_header(skip, next_block); const auto& global_props = get_global_properties(); - const auto& dynamic_global_props = get(dynamic_global_property_id_type()); + const auto& dynamic_global_props = get_dynamic_global_properties(); bool maint_needed = (dynamic_global_props.next_maintenance_time <= next_block.timestamp); _current_block_num = next_block_num; @@ -600,6 +600,8 @@ void database::_apply_block( const signed_block& next_block ) _current_op_in_trx = 0; _current_virtual_op = 0; + _issue_453_affected_assets.clear(); + for( const auto& trx : next_block.transactions ) { /* We do not need to push the undo state for each transaction @@ -639,6 +641,7 @@ void database::_apply_block( const signed_block& next_block ) clear_expired_proposals(); clear_expired_orders(); update_expired_feeds(); + update_core_exchange_rates(); update_withdraw_permissions(); update_tournaments(); update_betting_markets(next_block.timestamp); diff --git a/libraries/chain/db_market.cpp b/libraries/chain/db_market.cpp index 59f77762..77acedd3 100644 --- a/libraries/chain/db_market.cpp +++ b/libraries/chain/db_market.cpp @@ -426,14 +426,18 @@ bool database::fill_order(const force_settlement_object& settle, const asset& pa * * @return true if a margin call was executed. */ -bool database::check_call_orders(const asset_object& mia, bool enable_black_swan) +bool database::check_call_orders( const asset_object& mia, bool enable_black_swan, bool for_new_limit_order, + const asset_bitasset_data_object* bitasset_ptr ) { try { + static const auto& dyn_prop = get_dynamic_global_properties(); if( !mia.is_market_issued() ) return false; + auto maint_time = dyn_prop.next_maintenance_time; - if( check_for_blackswan( mia, enable_black_swan ) ) + const asset_bitasset_data_object& bitasset = ( bitasset_ptr ? *bitasset_ptr : mia.bitasset_data(*this) ); + + if( check_for_blackswan( mia, enable_black_swan, &bitasset ) ) return false; - const asset_bitasset_data_object& bitasset = mia.bitasset_data(*this); if( bitasset.is_prediction_market ) return false; if( bitasset.current_feed.settlement_price.is_null() ) return false; @@ -464,7 +468,13 @@ bool database::check_call_orders(const asset_object& mia, bool enable_black_swan bool filled_limit = false; bool margin_called = false; - while( !check_for_blackswan( mia, enable_black_swan ) && call_itr != call_end ) + auto head_time = head_block_time(); + auto head_num = head_block_num(); + + bool before_hardfork_615 = ( head_time < HARDFORK_615_TIME ); + bool after_hardfork_436 = ( head_time > HARDFORK_436_TIME ); + + while( !check_for_blackswan( mia, enable_black_swan, &bitasset ) && call_itr != call_end ) { bool filled_call = false; price match_price; @@ -506,7 +516,8 @@ bool database::check_call_orders(const asset_object& mia, bool enable_black_swan if( usd_to_buy * match_price > call_itr->get_collateral() ) { - elog( "black swan detected" ); + elog( "black swan detected on asset ${symbol} (${id}) at block ${b}", + ("id",mia.id)("symbol",mia.symbol)("b",head_num) ); edump((enable_black_swan)); FC_ASSERT( enable_black_swan ); globally_settle_asset(mia, bitasset.current_feed.settlement_price ); diff --git a/libraries/chain/db_update.cpp b/libraries/chain/db_update.cpp index 66825982..eb4acd0d 100644 --- a/libraries/chain/db_update.cpp +++ b/libraries/chain/db_update.cpp @@ -121,6 +121,7 @@ void database::update_last_irreversible_block() const global_property_object& gpo = get_global_properties(); const dynamic_global_property_object& dpo = get_dynamic_global_properties(); + // TODO for better performance, move this to db_maint, because only need to do it once per maintenance interval vector< const witness_object* > wit_objs; wit_objs.reserve( gpo.active_witnesses.size() ); for( const witness_id_type& wid : gpo.active_witnesses ) @@ -237,11 +238,12 @@ void database::clear_expired_proposals() * * A black swan occurs if MAX(HB,SP) <= LC */ -bool database::check_for_blackswan( const asset_object& mia, bool enable_black_swan ) +bool database::check_for_blackswan( const asset_object& mia, bool enable_black_swan, + const asset_bitasset_data_object* bitasset_ptr ) { if( !mia.is_market_issued() ) return false; - const asset_bitasset_data_object& bitasset = mia.bitasset_data(*this); + const asset_bitasset_data_object& bitasset = ( bitasset_ptr ? *bitasset_ptr : mia.bitasset_data(*this) ); if( bitasset.has_settlement() ) return true; // already force settled auto settle_price = bitasset.current_feed.settlement_price; if( settle_price.is_null() ) return false; // no feed @@ -466,40 +468,84 @@ void database::clear_expired_orders() void database::update_expired_feeds() { - const auto head_num = head_block_num(); const auto head_time = head_block_time(); - bool before_hf_615 = ( head_time < HARDFORK_615_TIME ); - auto& asset_idx = get_index_type().indices().get(); - auto itr = asset_idx.lower_bound( true /** market issued */ ); - while( itr != asset_idx.end() ) - { - const asset_object& a = *itr; - ++itr; - assert( a.is_market_issued() ); + bool after_hardfork_615 = ( head_time >= HARDFORK_615_TIME ); - const asset_bitasset_data_object& b = a.bitasset_data(*this); - bool feed_is_expired; - if( before_hf_615 ) - feed_is_expired = b.feed_is_expired_before_hardfork_615( head_time ); - else - feed_is_expired = b.feed_is_expired( head_time ); - if( feed_is_expired ) + const auto& idx = get_index_type().indices().get(); + auto itr = idx.begin(); + while( itr != idx.end() && itr->feed_is_expired( head_time ) ) + { + const asset_bitasset_data_object& b = *itr; + ++itr; // not always process begin() because old code skipped updating some assets before hf 615 + bool update_cer = false; // for better performance, to only update bitasset once, also check CER in this function + const asset_object* asset_ptr = nullptr; + // update feeds, check margin calls + if( after_hardfork_615 || b.feed_is_expired_before_hardfork_615( head_time ) ) { - modify(b, [head_time](asset_bitasset_data_object& a) { - a.update_median_feeds(head_time); - }); - bool called_some = check_call_orders(b.current_feed.settlement_price.base.asset_id(*this)); - if( called_some && before_hf_615 ) + auto old_median_feed = b.current_feed; + modify( b, [head_time,&update_cer]( asset_bitasset_data_object& abdo ) { - wlog( "Graphene issue #615: called some for asset ${a} on block #${b}, feed really expired: ${f}", - ("a", a.symbol) ("b", head_num) ("f", b.feed_is_expired(head_time)) ); + abdo.update_median_feeds( head_time ); + if( abdo.need_to_update_cer() ) + { + update_cer = true; + abdo.asset_cer_updated = false; + abdo.feed_cer_updated = false; + } + }); + if( !b.current_feed.settlement_price.is_null() && !( b.current_feed == old_median_feed ) ) // `==` check is safe here + { + asset_ptr = &b.asset_id( *this ); + check_call_orders( *asset_ptr, true, false, &b ); } } - if( !b.current_feed.core_exchange_rate.is_null() && - a.options.core_exchange_rate != b.current_feed.core_exchange_rate ) - modify(a, [&b](asset_object& a) { - a.options.core_exchange_rate = b.current_feed.core_exchange_rate; + // update CER + if( update_cer ) + { + if( !asset_ptr ) + asset_ptr = &b.asset_id( *this ); + if( asset_ptr->options.core_exchange_rate != b.current_feed.core_exchange_rate ) + { + modify( *asset_ptr, [&b]( asset_object& ao ) + { + ao.options.core_exchange_rate = b.current_feed.core_exchange_rate; + }); + } + } + } // for each asset whose feed is expired + + // process assets affected by bitshares-core issue 453 before hard fork 615 + if( !after_hardfork_615 ) + { + for( asset_id_type a : _issue_453_affected_assets ) + { + check_call_orders( a(*this) ); + } + } +} + +void database::update_core_exchange_rates() +{ + const auto& idx = get_index_type().indices().get(); + if( idx.begin() != idx.end() ) + { + for( auto itr = idx.rbegin(); itr->need_to_update_cer(); itr = idx.rbegin() ) + { + const asset_bitasset_data_object& b = *itr; + const asset_object& a = b.asset_id( *this ); + if( a.options.core_exchange_rate != b.current_feed.core_exchange_rate ) + { + modify( a, [&b]( asset_object& ao ) + { + ao.options.core_exchange_rate = b.current_feed.core_exchange_rate; + }); + } + modify( b, []( asset_bitasset_data_object& abdo ) + { + abdo.asset_cer_updated = false; + abdo.feed_cer_updated = false; }); + } } } diff --git a/libraries/chain/include/graphene/chain/asset_object.hpp b/libraries/chain/include/graphene/chain/asset_object.hpp index f1df4681..cba33bb8 100644 --- a/libraries/chain/include/graphene/chain/asset_object.hpp +++ b/libraries/chain/include/graphene/chain/asset_object.hpp @@ -193,6 +193,9 @@ namespace graphene { namespace chain { static const uint8_t space_id = implementation_ids; static const uint8_t type_id = impl_asset_bitasset_data_type; + /// The asset this object belong to + asset_id_type asset_id; + /// The tunable options for BitAssets are stored in this field. bitasset_options options; @@ -230,6 +233,18 @@ namespace graphene { namespace chain { share_type settlement_fund; ///@} + /// Track whether core_exchange_rate in corresponding asset_object has updated + bool asset_cer_updated = false; + + /// Track whether core exchange rate in current feed has updated + bool feed_cer_updated = false; + + /// Whether need to update core_exchange_rate in asset_object + bool need_to_update_cer() const + { + return ( ( feed_cer_updated || asset_cer_updated ) && !current_feed.core_exchange_rate.is_null() ); + } + /// The time when @ref current_feed would expire time_point_sec feed_expiration_time()const { @@ -247,14 +262,34 @@ namespace graphene { namespace chain { void update_median_feeds(time_point_sec current_time); }; + // key extractor for short backing asset + struct bitasset_short_backing_asset_extractor + { + typedef asset_id_type result_type; + result_type operator() (const asset_bitasset_data_object& obj) const + { + return obj.options.short_backing_asset; + } + }; + + struct by_short_backing_asset; struct by_feed_expiration; + struct by_cer_update; + typedef multi_index_container< asset_bitasset_data_object, indexed_by< - ordered_unique< tag, member< object, object_id_type, &object::id > >, - ordered_non_unique< tag, - const_mem_fun< asset_bitasset_data_object, time_point_sec, &asset_bitasset_data_object::feed_expiration_time > - > + ordered_unique< tag, member< object, object_id_type, &object::id > >, + ordered_non_unique< tag, bitasset_short_backing_asset_extractor >, + ordered_unique< tag, + composite_key< asset_bitasset_data_object, + const_mem_fun< asset_bitasset_data_object, time_point_sec, &asset_bitasset_data_object::feed_expiration_time >, + member< asset_bitasset_data_object, asset_id_type, &asset_bitasset_data_object::asset_id > + > + >, + ordered_non_unique< tag, + const_mem_fun< asset_bitasset_data_object, bool, &asset_bitasset_data_object::need_to_update_cer > + > > > asset_bitasset_data_object_multi_index_type; //typedef flat_index asset_bitasset_data_index; @@ -481,6 +516,7 @@ FC_REFLECT_DERIVED( graphene::chain::asset_dynamic_data_object, (graphene::db::o (current_supply)(sweeps_tickets_sold)(confidential_supply)(accumulated_fees)(fee_pool) ) FC_REFLECT_DERIVED( graphene::chain::asset_bitasset_data_object, (graphene::db::object), + (asset_id) (feeds) (current_feed) (current_feed_publication_time) @@ -489,6 +525,8 @@ FC_REFLECT_DERIVED( graphene::chain::asset_bitasset_data_object, (graphene::db:: (is_prediction_market) (settlement_price) (settlement_fund) + (asset_cer_updated) + (feed_cer_updated) ) FC_REFLECT_DERIVED( graphene::chain::asset_dividend_data_object, (graphene::db::object), diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp index 52cfdd8b..28e84c28 100644 --- a/libraries/chain/include/graphene/chain/database.hpp +++ b/libraries/chain/include/graphene/chain/database.hpp @@ -437,7 +437,8 @@ namespace graphene { namespace chain { bool fill_order( const call_order_object& order, const asset& pays, const asset& receives ); bool fill_order( const force_settlement_object& settle, const asset& pays, const asset& receives ); - bool check_call_orders( const asset_object& mia, bool enable_black_swan = true ); + bool check_call_orders( const asset_object& mia, bool enable_black_swan = true, bool for_new_limit_order = false, + const asset_bitasset_data_object* bitasset_ptr = nullptr ); // helpers to fill_order void pay_order( const account_object& receiver, const asset& receives, const asset& pays ); @@ -505,11 +506,13 @@ namespace graphene { namespace chain { void clear_expired_proposals(); void clear_expired_orders(); void update_expired_feeds(); + void update_core_exchange_rates(); void update_maintenance_flag( bool new_maintenance_flag ); void update_withdraw_permissions(); void update_tournaments(); void update_betting_markets(fc::time_point_sec current_block_time); - bool check_for_blackswan( const asset_object& mia, bool enable_black_swan = true ); + bool check_for_blackswan( const asset_object& mia, bool enable_black_swan = true, + const asset_bitasset_data_object* bitasset_ptr = nullptr ); ///Steps performed only at maintenance intervals ///@{ @@ -583,6 +586,8 @@ namespace graphene { namespace chain { * database::close() has not been called, or failed during execution. */ bool _opened = false; + /// Tracks assets affected by bitshares-core issue #453 before hard fork #615 in one block + flat_set _issue_453_affected_assets; }; namespace detail