Improve update_expired_feeds performance #1093

libraries/chain/asset_evaluator.cpp

@@ -149,15 +149,16 @@ object_id_type asset_create_evaluator::do_apply( const asset_create_operation& o
         });
     }
 
+   auto next_asset_id = db().get_index_type<asset_index>().get_next_id();
+
    asset_bitasset_data_id_type bit_asset_id;
    if( op.bitasset_opts.valid() )
       bit_asset_id = db().create<asset_bitasset_data_object>( [&]( asset_bitasset_data_object& a ) {
             a.options = *op.bitasset_opts;
             a.is_prediction_market = op.is_prediction_market;
+            a.asset_id = next_asset_id;
          }).id;
 
-   auto next_asset_id = db().get_index_type<asset_index>().get_next_id();
-
    const asset_object& new_asset =
      db().create<asset_object>( [&]( asset_object& a ) {
          a.issuer = op.issuer;
@@ -297,17 +298,18 @@ object_id_type lottery_asset_create_evaluator::do_apply( const lottery_asset_cre
          });
       }
 
+   auto next_asset_id = db().get_index_type<asset_index>().get_next_id();
+
    asset_bitasset_data_id_type bit_asset_id;
    if( op.bitasset_opts.valid() )
       bit_asset_id = db().create<asset_bitasset_data_object>( [&op,next_asset_id]( asset_bitasset_data_object& a ) {
             a.options = *op.bitasset_opts;
             a.is_prediction_market = op.is_prediction_market;
+            a.asset_id = next_asset_id;
          }).id;
 
-   auto next_asset_id = db().get_index_type<asset_index>().get_next_id();
-
    const asset_object& new_asset =
-     db().create<asset_object>( [&op,next_asset_id,&dyn_asset,bit_asset_id]( asset_object& a ) {
+     db().create<asset_object>( [&op,next_asset_id,&dyn_asset,bit_asset_id,this]( asset_object& a ) {
          a.issuer = op.issuer;
          a.symbol = op.symbol;
          a.precision = op.precision;
@@ -483,6 +485,20 @@ void_result asset_update_evaluator::do_apply(const asset_update_operation& o)
          d.cancel_order(*itr);
    }
 
+   // For market-issued assets, if core change rate changed, update flag in bitasset data
+   if( asset_to_update->is_market_issued()
+          && asset_to_update->options.core_exchange_rate != o.new_options.core_exchange_rate )
+   {
+      const auto& bitasset = asset_to_update->bitasset_data(d);
+      if( !bitasset.asset_cer_updated )
+      {
+         d.modify( bitasset, [](asset_bitasset_data_object& b)
+         {
+            b.asset_cer_updated = true;
+         });
+      }
+   }
+
    d.modify(*asset_to_update, [&o](asset_object& a) {
       if( o.new_issuer )
          a.issuer = *o.new_issuer;

libraries/chain/asset_object.cpp

@@ -61,12 +61,15 @@ void asset_bitasset_data_object::update_median_feeds(time_point_sec current_time
    if( current_feeds.size() < options.minimum_feeds )
    {
       //... don't calculate a median, and set a null feed
+      feed_cer_updated = false; // new median cer is null, won't update asset_object anyway, set to false for better performance
       current_feed_publication_time = current_time;
       current_feed = price_feed();
       return;
    }
    if( current_feeds.size() == 1 )
    {
+      if( current_feed.core_exchange_rate != current_feeds.front().get().core_exchange_rate )
+         feed_cer_updated = true;
       current_feed = std::move(current_feeds.front());
       return;
    }
@@ -85,6 +88,8 @@ void asset_bitasset_data_object::update_median_feeds(time_point_sec current_time
 #undef CALCULATE_MEDIAN_VALUE
    // *** End Median Calculations ***
 
+   if( current_feed.core_exchange_rate != median_feed.core_exchange_rate )
+      feed_cer_updated = true;
    current_feed = median_feed;
 }
 

libraries/chain/db_block.cpp

@@ -592,7 +592,7 @@ void database::_apply_block( const signed_block& next_block )
 
    const witness_object& signing_witness = validate_block_header(skip, next_block);
    const auto& global_props = get_global_properties();
-   const auto& dynamic_global_props = get<dynamic_global_property_object>(dynamic_global_property_id_type());
+   const auto& dynamic_global_props = get_dynamic_global_properties();
    bool maint_needed = (dynamic_global_props.next_maintenance_time <= next_block.timestamp);
 
    _current_block_num    = next_block_num;
@@ -600,6 +600,8 @@ void database::_apply_block( const signed_block& next_block )
    _current_op_in_trx    = 0;
    _current_virtual_op   = 0;
 
+   _issue_453_affected_assets.clear();
+
    for( const auto& trx : next_block.transactions )
    {
       /* We do not need to push the undo state for each transaction
@@ -639,6 +641,7 @@ void database::_apply_block( const signed_block& next_block )
    clear_expired_proposals();
    clear_expired_orders();
    update_expired_feeds();
+   update_core_exchange_rates();
    update_withdraw_permissions();
    update_tournaments();
    update_betting_markets(next_block.timestamp);

libraries/chain/db_market.cpp

@@ -426,14 +426,18 @@ bool database::fill_order(const force_settlement_object& settle, const asset& pa
  *
  *  @return true if a margin call was executed.
  */
-bool database::check_call_orders(const asset_object& mia, bool enable_black_swan)
+bool database::check_call_orders( const asset_object& mia, bool enable_black_swan, bool for_new_limit_order,
+                                  const asset_bitasset_data_object* bitasset_ptr )
 { try {
+    static const auto& dyn_prop = get_dynamic_global_properties();
     if( !mia.is_market_issued() ) return false;
+    auto maint_time = dyn_prop.next_maintenance_time;
 
-    if( check_for_blackswan( mia, enable_black_swan ) ) 
+    const asset_bitasset_data_object& bitasset = ( bitasset_ptr ? *bitasset_ptr : mia.bitasset_data(*this) );
+
+    if( check_for_blackswan( mia, enable_black_swan, &bitasset ) )
        return false;
 
-    const asset_bitasset_data_object& bitasset = mia.bitasset_data(*this);
     if( bitasset.is_prediction_market ) return false;
     if( bitasset.current_feed.settlement_price.is_null() ) return false;
 
@@ -464,7 +468,13 @@ bool database::check_call_orders(const asset_object& mia, bool enable_black_swan
     bool filled_limit = false;
     bool margin_called = false;
 
-    while( !check_for_blackswan( mia, enable_black_swan ) && call_itr != call_end )
+    auto head_time = head_block_time();
+    auto head_num = head_block_num();
+
+    bool before_hardfork_615 = ( head_time < HARDFORK_615_TIME );
+    bool after_hardfork_436 = ( head_time > HARDFORK_436_TIME );
+
+    while( !check_for_blackswan( mia, enable_black_swan, &bitasset ) && call_itr != call_end )
     {
        bool  filled_call      = false;
        price match_price;
@@ -506,7 +516,8 @@ bool database::check_call_orders(const asset_object& mia, bool enable_black_swan
 
        if( usd_to_buy * match_price > call_itr->get_collateral() )
        {
-          elog( "black swan detected" ); 
+          elog( "black swan detected on asset ${symbol} (${id}) at block ${b}",
+                ("id",mia.id)("symbol",mia.symbol)("b",head_num) );
           edump((enable_black_swan));
           FC_ASSERT( enable_black_swan );
           globally_settle_asset(mia, bitasset.current_feed.settlement_price );

libraries/chain/db_update.cpp

@@ -121,6 +121,7 @@ void database::update_last_irreversible_block()
    const global_property_object& gpo = get_global_properties();
    const dynamic_global_property_object& dpo = get_dynamic_global_properties();
 
+   // TODO for better performance, move this to db_maint, because only need to do it once per maintenance interval
    vector< const witness_object* > wit_objs;
    wit_objs.reserve( gpo.active_witnesses.size() );
    for( const witness_id_type& wid : gpo.active_witnesses )
@@ -237,11 +238,12 @@ void database::clear_expired_proposals()
  *
  *  A black swan occurs if MAX(HB,SP) <= LC
  */
-bool database::check_for_blackswan( const asset_object& mia, bool enable_black_swan )
+bool database::check_for_blackswan( const asset_object& mia, bool enable_black_swan,
+                                    const asset_bitasset_data_object* bitasset_ptr )
 {
     if( !mia.is_market_issued() ) return false;
 
-    const asset_bitasset_data_object& bitasset = mia.bitasset_data(*this);
+    const asset_bitasset_data_object& bitasset = ( bitasset_ptr ? *bitasset_ptr : mia.bitasset_data(*this) );
     if( bitasset.has_settlement() ) return true; // already force settled
     auto settle_price = bitasset.current_feed.settlement_price;
     if( settle_price.is_null() ) return false; // no feed
@@ -466,40 +468,84 @@ void database::clear_expired_orders()
 
 void database::update_expired_feeds()
 {
-   const auto head_num = head_block_num();
    const auto head_time = head_block_time();
-   bool before_hf_615 = ( head_time < HARDFORK_615_TIME );
+   bool after_hardfork_615 = ( head_time >= HARDFORK_615_TIME );
-   auto& asset_idx = get_index_type<asset_index>().indices().get<by_type>();
-   auto itr = asset_idx.lower_bound( true /** market issued */ );
-   while( itr != asset_idx.end() )
-   {
-      const asset_object& a = *itr;
-      ++itr;
-      assert( a.is_market_issued() );
 
-      const asset_bitasset_data_object& b = a.bitasset_data(*this);
+   const auto& idx = get_index_type<asset_bitasset_data_index>().indices().get<by_feed_expiration>();
-      bool feed_is_expired;
+   auto itr = idx.begin();
-      if( before_hf_615 )
+   while( itr != idx.end() && itr->feed_is_expired( head_time ) )
-         feed_is_expired = b.feed_is_expired_before_hardfork_615( head_time );
+   {
-      else
+      const asset_bitasset_data_object& b = *itr;
-         feed_is_expired = b.feed_is_expired( head_time );
+      ++itr; // not always process begin() because old code skipped updating some assets before hf 615
-      if( feed_is_expired )
+      bool update_cer = false; // for better performance, to only update bitasset once, also check CER in this function
+      const asset_object* asset_ptr = nullptr;
+      // update feeds, check margin calls
+      if( after_hardfork_615 || b.feed_is_expired_before_hardfork_615( head_time ) )
       {
-         modify(b, [head_time](asset_bitasset_data_object& a) {
+         auto old_median_feed = b.current_feed;
-            a.update_median_feeds(head_time);
+         modify( b, [head_time,&update_cer]( asset_bitasset_data_object& abdo )
-         });
-         bool called_some = check_call_orders(b.current_feed.settlement_price.base.asset_id(*this));
-         if( called_some && before_hf_615 )
          {
-            wlog( "Graphene issue #615: called some for asset ${a} on block #${b}, feed really expired: ${f}",
+            abdo.update_median_feeds( head_time );
-                  ("a", a.symbol) ("b", head_num) ("f", b.feed_is_expired(head_time)) );
+            if( abdo.need_to_update_cer() )
+            {
+               update_cer = true;
+               abdo.asset_cer_updated = false;
+               abdo.feed_cer_updated = false;
+            }
+         });
+         if( !b.current_feed.settlement_price.is_null() && !( b.current_feed == old_median_feed ) ) // `==` check is safe here
+         {
+            asset_ptr = &b.asset_id( *this );
+            check_call_orders( *asset_ptr, true, false, &b );
          }
       }
-      if( !b.current_feed.core_exchange_rate.is_null() &&
+      // update CER
-          a.options.core_exchange_rate != b.current_feed.core_exchange_rate )
+      if( update_cer )
-         modify(a, [&b](asset_object& a) {
+      {
-            a.options.core_exchange_rate = b.current_feed.core_exchange_rate;
+         if( !asset_ptr )
+            asset_ptr = &b.asset_id( *this );
+         if( asset_ptr->options.core_exchange_rate != b.current_feed.core_exchange_rate )
+         {
+            modify( *asset_ptr, [&b]( asset_object& ao )
+            {
+               ao.options.core_exchange_rate = b.current_feed.core_exchange_rate;
+            });
+         }
+      }
+   } // for each asset whose feed is expired
+
+   // process assets affected by bitshares-core issue 453 before hard fork 615
+   if( !after_hardfork_615 )
+   {
+      for( asset_id_type a : _issue_453_affected_assets )
+      {
+         check_call_orders( a(*this) );
+      }
+   }
+}
+
+void database::update_core_exchange_rates()
+{
+   const auto& idx = get_index_type<asset_bitasset_data_index>().indices().get<by_cer_update>();
+   if( idx.begin() != idx.end() )
+   {
+      for( auto itr = idx.rbegin(); itr->need_to_update_cer(); itr = idx.rbegin() )
+      {
+         const asset_bitasset_data_object& b = *itr;
+         const asset_object& a = b.asset_id( *this );
+         if( a.options.core_exchange_rate != b.current_feed.core_exchange_rate )
+         {
+            modify( a, [&b]( asset_object& ao )
+            {
+               ao.options.core_exchange_rate = b.current_feed.core_exchange_rate;
+            });
+         }
+         modify( b, []( asset_bitasset_data_object& abdo )
+         {
+            abdo.asset_cer_updated = false;
+            abdo.feed_cer_updated = false;
          });
+      }
    }
 }
 

libraries/chain/include/graphene/chain/asset_object.hpp

@@ -193,6 +193,9 @@ namespace graphene { namespace chain {
          static const uint8_t space_id = implementation_ids;
          static const uint8_t type_id  = impl_asset_bitasset_data_type;
 
+         /// The asset this object belong to
+         asset_id_type asset_id;
+
          /// The tunable options for BitAssets are stored in this field.
          bitasset_options options;
 
@@ -230,6 +233,18 @@ namespace graphene { namespace chain {
          share_type settlement_fund;
          ///@}
 
+         /// Track whether core_exchange_rate in corresponding asset_object has updated
+         bool asset_cer_updated = false;
+
+         /// Track whether core exchange rate in current feed has updated
+         bool feed_cer_updated = false;
+
+         /// Whether need to update core_exchange_rate in asset_object
+         bool need_to_update_cer() const
+         {
+            return ( ( feed_cer_updated || asset_cer_updated ) && !current_feed.core_exchange_rate.is_null() );
+         }
+
          /// The time when @ref current_feed would expire
          time_point_sec feed_expiration_time()const
          {
@@ -247,14 +262,34 @@ namespace graphene { namespace chain {
          void update_median_feeds(time_point_sec current_time);
    };
 
+   // key extractor for short backing asset
+   struct bitasset_short_backing_asset_extractor
+   {
+      typedef asset_id_type result_type;
+      result_type operator() (const asset_bitasset_data_object& obj) const
+      {
+         return obj.options.short_backing_asset;
+      }
+   };
+
+   struct by_short_backing_asset;
    struct by_feed_expiration;
+   struct by_cer_update;
+
    typedef multi_index_container<
       asset_bitasset_data_object,
       indexed_by<
-         ordered_unique< tag<by_id>, member< object, object_id_type, &object::id > >,
+           ordered_unique< tag<by_id>, member< object, object_id_type, &object::id > >,
-         ordered_non_unique< tag<by_feed_expiration>,
+           ordered_non_unique< tag<by_short_backing_asset>, bitasset_short_backing_asset_extractor >,
-            const_mem_fun< asset_bitasset_data_object, time_point_sec, &asset_bitasset_data_object::feed_expiration_time >
+           ordered_unique< tag<by_feed_expiration>,
-         >
+              composite_key< asset_bitasset_data_object,
+                 const_mem_fun< asset_bitasset_data_object, time_point_sec, &asset_bitasset_data_object::feed_expiration_time >,
+                 member< asset_bitasset_data_object, asset_id_type, &asset_bitasset_data_object::asset_id >
+              >
+           >,
+           ordered_non_unique< tag<by_cer_update>,
+                               const_mem_fun< asset_bitasset_data_object, bool, &asset_bitasset_data_object::need_to_update_cer >
+           >
       >
    > asset_bitasset_data_object_multi_index_type;
    //typedef flat_index<asset_bitasset_data_object> asset_bitasset_data_index;
@@ -481,6 +516,7 @@ FC_REFLECT_DERIVED( graphene::chain::asset_dynamic_data_object, (graphene::db::o
                     (current_supply)(sweeps_tickets_sold)(confidential_supply)(accumulated_fees)(fee_pool) )
 
 FC_REFLECT_DERIVED( graphene::chain::asset_bitasset_data_object, (graphene::db::object),
+                    (asset_id)
                     (feeds)
                     (current_feed)
                     (current_feed_publication_time)
@@ -489,6 +525,8 @@ FC_REFLECT_DERIVED( graphene::chain::asset_bitasset_data_object, (graphene::db::
                     (is_prediction_market)
                     (settlement_price)
                     (settlement_fund)
+                    (asset_cer_updated)
+                    (feed_cer_updated)
                   )
    
 FC_REFLECT_DERIVED( graphene::chain::asset_dividend_data_object, (graphene::db::object),

libraries/chain/include/graphene/chain/database.hpp

@@ -437,7 +437,8 @@ namespace graphene { namespace chain {
          bool fill_order( const call_order_object& order, const asset& pays, const asset& receives );
          bool fill_order( const force_settlement_object& settle, const asset& pays, const asset& receives );
 
-         bool check_call_orders( const asset_object& mia, bool enable_black_swan = true );
+         bool check_call_orders( const asset_object& mia, bool enable_black_swan = true, bool for_new_limit_order = false,
+                                 const asset_bitasset_data_object* bitasset_ptr = nullptr );
 
          // helpers to fill_order
          void pay_order( const account_object& receiver, const asset& receives, const asset& pays );
@@ -505,11 +506,13 @@ namespace graphene { namespace chain {
          void clear_expired_proposals();
          void clear_expired_orders();
          void update_expired_feeds();
+         void update_core_exchange_rates();
          void update_maintenance_flag( bool new_maintenance_flag );
          void update_withdraw_permissions();
          void update_tournaments();
          void update_betting_markets(fc::time_point_sec current_block_time);
-         bool check_for_blackswan( const asset_object& mia, bool enable_black_swan = true );
+         bool check_for_blackswan( const asset_object& mia, bool enable_black_swan = true,
+                                   const asset_bitasset_data_object* bitasset_ptr = nullptr );
 
          ///Steps performed only at maintenance intervals
          ///@{
@@ -583,6 +586,8 @@ namespace graphene { namespace chain {
           * database::close() has not been called, or failed during execution.
           */
          bool                              _opened = false;
+         /// Tracks assets affected by bitshares-core issue #453 before hard fork #615 in one block
+         flat_set<asset_id_type>           _issue_453_affected_assets;
    };
 
    namespace detail