improve reindexing performance

libraries/CMakeLists.txt

@@ -4,6 +4,7 @@ add_subdirectory( deterministic_openssl_rand )
 add_subdirectory( chain )
 add_subdirectory( egenesis )
 add_subdirectory( net )
+add_subdirectory( p2p )
 add_subdirectory( time )
 add_subdirectory( utilities )
 add_subdirectory( app )

libraries/chain/db_block.cpp

@@ -439,6 +439,8 @@ void database::_apply_block( const signed_block& next_block )
 } FC_CAPTURE_AND_RETHROW( (next_block.block_num()) )  }
 
 void database::notify_changed_objects()
+{ try {
+   if( _undo_db.enabled() ) 
    {
       const auto& head_undo = _undo_db.head();
       vector<object_id_type> changed_ids;  changed_ids.reserve(head_undo.old_values.size());
@@ -453,6 +455,7 @@ void database::notify_changed_objects()
       }
       changed_objects(changed_ids);
    }
+} FC_CAPTURE_AND_RETHROW() }
 
 processed_transaction database::apply_transaction(const signed_transaction& trx, uint32_t skip)
 {

libraries/chain/db_management.cpp

@@ -38,6 +38,7 @@ database::~database(){
 
 void database::reindex(fc::path data_dir, const genesis_state_type& initial_allocation)
 { try {
+   ilog( "reindexing blockchain" );
    wipe(data_dir, false);
    open(data_dir, [&initial_allocation]{return initial_allocation;});
 
@@ -47,19 +48,21 @@ void database::reindex(fc::path data_dir, const genesis_state_type& initial_allo
 
    const auto last_block_num = last_block->block_num();
 
+   ilog( "Replaying blocks..." );
    // TODO: disable undo tracking during reindex, this currently causes crashes in the benchmark test
-   //_undo_db.disable();
+   _undo_db.disable();
    for( uint32_t i = 1; i <= last_block_num; ++i )
    {
+      if( i % 1000 == 0 ) std::cerr << "   " << double(i*100)/last_block_num << "%      \r";
       apply_block(*_block_id_to_block.fetch_by_number(i), skip_witness_signature |
                                 skip_transaction_signatures |
                                 skip_transaction_dupe_check |
                                 skip_tapos_check |
                                 skip_authority_check);
    }
-   //_undo_db.enable();
+   _undo_db.enable();
    auto end = fc::time_point::now();
-   wdump( ((end-start).count()/1000000.0) );
+   ilog( "Done reindexing, elapsed time: ${t} sec", ("t",double((end-start).count())/1000000.0 ) );
 } FC_CAPTURE_AND_RETHROW( (data_dir) ) }
 
 void database::wipe(const fc::path& data_dir, bool include_blocks)

libraries/chain/include/graphene/chain/witness_scheduler.hpp

@@ -26,7 +26,7 @@
 
 namespace graphene { namespace chain {
 
-using boost::container::flat_set;
+//using boost::container::flat_set;
 
 enum witness_scheduler_relax_flags
 {
@@ -50,7 +50,7 @@ class generic_witness_scheduler
          assert( _turns == turns   );
 #endif
 
-         flat_set< WitnessID > witness_set;
+         set< WitnessID > witness_set;
          // make sure each witness_id occurs only once among the three states
          auto process_id = [&]( WitnessID item )
          {
@@ -249,14 +249,15 @@ class generic_witness_scheduler
       template< typename T >
       void update( const T& revised_set )
       {
-         flat_set< WitnessID > current_set;
+         set< WitnessID > current_set;
-         flat_set< WitnessID > schedule_set;
+         set< WitnessID > schedule_set;
 
-         current_set.reserve(
+       /*  current_set.reserve(
               _ineligible_waiting_for_token.size()
             + _ineligible_no_turn.size()
             + _eligible.size()
             + _schedule.size() );
+            */
          for( const auto& item : _ineligible_waiting_for_token )
             current_set.insert( item.first );
          for( const WitnessID& item : _ineligible_no_turn )
@@ -269,16 +270,16 @@ class generic_witness_scheduler
             schedule_set.insert( item );
          }
 
-         flat_set< WitnessID > insertion_set;
+         set< WitnessID > insertion_set;
-         insertion_set.reserve( revised_set.size() );
+         //insertion_set.reserve( revised_set.size() );
          for( const WitnessID& item : revised_set )
          {
             if( current_set.find( item ) == current_set.end() )
                insertion_set.insert( item );
          }
 
-         flat_set< WitnessID > removal_set;
+         set< WitnessID > removal_set;
-         removal_set.reserve( current_set.size() );
+         //removal_set.reserve( current_set.size() );
          for( const WitnessID& item : current_set )
          {
             if( revised_set.find( item ) == revised_set.end() )
@@ -366,7 +367,7 @@ class generic_witness_scheduler
       std::deque < WitnessID > _schedule;
 
       // in _schedule, but not to be replaced
-      flat_set< WitnessID > _lame_duck;
+      set< WitnessID > _lame_duck;
 };
 
 template< typename WitnessID, typename RNG, typename CountType, typename OffsetType, bool debug = true >

libraries/db/include/graphene/db/undo_database.hpp

@@ -85,6 +85,7 @@ namespace graphene { namespace db {
 
          void    disable();
          void    enable();
+         bool    enabled()const { return !_disabled; }
 
          session start_undo_session();
          /**

libraries/net/include/graphene/net/message_oriented_connection.hpp

@@ -40,6 +40,7 @@ namespace graphene { namespace net {
        message_oriented_connection(message_oriented_connection_delegate* delegate = nullptr);
        ~message_oriented_connection();
        fc::tcp_socket& get_socket();
+
        void accept();
        void bind(const fc::ip::endpoint& local_endpoint);
        void connect_to(const fc::ip::endpoint& remote_endpoint);