From 2464b788adf0e8b9cb909746095848e499fa7520 Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Mon, 24 Aug 2015 17:57:44 -0400 Subject: [PATCH] improve reindexing performance --- libraries/CMakeLists.txt | 1 + libraries/chain/db_block.cpp | 27 ++++++++------ libraries/chain/db_management.cpp | 9 +++-- .../graphene/chain/witness_scheduler.hpp | 21 ++++++----- .../db/include/graphene/db/undo_database.hpp | 1 + .../net/message_oriented_connection.hpp | 37 ++++++++++--------- 6 files changed, 53 insertions(+), 43 deletions(-) diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt index 6af806f9..2a0754ef 100644 --- a/libraries/CMakeLists.txt +++ b/libraries/CMakeLists.txt @@ -4,6 +4,7 @@ add_subdirectory( deterministic_openssl_rand ) add_subdirectory( chain ) add_subdirectory( egenesis ) add_subdirectory( net ) +add_subdirectory( p2p ) add_subdirectory( time ) add_subdirectory( utilities ) add_subdirectory( app ) diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp index 62c77109..2cf8e9d1 100644 --- a/libraries/chain/db_block.cpp +++ b/libraries/chain/db_block.cpp @@ -439,20 +439,23 @@ void database::_apply_block( const signed_block& next_block ) } FC_CAPTURE_AND_RETHROW( (next_block.block_num()) ) } void database::notify_changed_objects() -{ - const auto& head_undo = _undo_db.head(); - vector changed_ids; changed_ids.reserve(head_undo.old_values.size()); - for( const auto& item : head_undo.old_values ) changed_ids.push_back(item.first); - for( const auto& item : head_undo.new_ids ) changed_ids.push_back(item); - vector removed; - removed.reserve( head_undo.removed.size() ); - for( const auto& item : head_undo.removed ) +{ try { + if( _undo_db.enabled() ) { - changed_ids.push_back( item.first ); - removed.emplace_back( item.second.get() ); + const auto& head_undo = _undo_db.head(); + vector changed_ids; changed_ids.reserve(head_undo.old_values.size()); + for( const auto& item : head_undo.old_values ) changed_ids.push_back(item.first); + for( const auto& item : head_undo.new_ids ) changed_ids.push_back(item); + vector removed; + removed.reserve( head_undo.removed.size() ); + for( const auto& item : head_undo.removed ) + { + changed_ids.push_back( item.first ); + removed.emplace_back( item.second.get() ); + } + changed_objects(changed_ids); } - changed_objects(changed_ids); -} +} FC_CAPTURE_AND_RETHROW() } processed_transaction database::apply_transaction(const signed_transaction& trx, uint32_t skip) { diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp index 15466734..a28d40d3 100644 --- a/libraries/chain/db_management.cpp +++ b/libraries/chain/db_management.cpp @@ -38,6 +38,7 @@ database::~database(){ void database::reindex(fc::path data_dir, const genesis_state_type& initial_allocation) { try { + ilog( "reindexing blockchain" ); wipe(data_dir, false); open(data_dir, [&initial_allocation]{return initial_allocation;}); @@ -47,19 +48,21 @@ void database::reindex(fc::path data_dir, const genesis_state_type& initial_allo const auto last_block_num = last_block->block_num(); + ilog( "Replaying blocks..." ); // TODO: disable undo tracking during reindex, this currently causes crashes in the benchmark test - //_undo_db.disable(); + _undo_db.disable(); for( uint32_t i = 1; i <= last_block_num; ++i ) { + if( i % 1000 == 0 ) std::cerr << " " << double(i*100)/last_block_num << "% \r"; apply_block(*_block_id_to_block.fetch_by_number(i), skip_witness_signature | skip_transaction_signatures | skip_transaction_dupe_check | skip_tapos_check | skip_authority_check); } - //_undo_db.enable(); + _undo_db.enable(); auto end = fc::time_point::now(); - wdump( ((end-start).count()/1000000.0) ); + ilog( "Done reindexing, elapsed time: ${t} sec", ("t",double((end-start).count())/1000000.0 ) ); } FC_CAPTURE_AND_RETHROW( (data_dir) ) } void database::wipe(const fc::path& data_dir, bool include_blocks) diff --git a/libraries/chain/include/graphene/chain/witness_scheduler.hpp b/libraries/chain/include/graphene/chain/witness_scheduler.hpp index ec595922..5feef417 100644 --- a/libraries/chain/include/graphene/chain/witness_scheduler.hpp +++ b/libraries/chain/include/graphene/chain/witness_scheduler.hpp @@ -26,7 +26,7 @@ namespace graphene { namespace chain { -using boost::container::flat_set; +//using boost::container::flat_set; enum witness_scheduler_relax_flags { @@ -50,7 +50,7 @@ class generic_witness_scheduler assert( _turns == turns ); #endif - flat_set< WitnessID > witness_set; + set< WitnessID > witness_set; // make sure each witness_id occurs only once among the three states auto process_id = [&]( WitnessID item ) { @@ -249,14 +249,15 @@ class generic_witness_scheduler template< typename T > void update( const T& revised_set ) { - flat_set< WitnessID > current_set; - flat_set< WitnessID > schedule_set; + set< WitnessID > current_set; + set< WitnessID > schedule_set; - current_set.reserve( + /* current_set.reserve( _ineligible_waiting_for_token.size() + _ineligible_no_turn.size() + _eligible.size() + _schedule.size() ); + */ for( const auto& item : _ineligible_waiting_for_token ) current_set.insert( item.first ); for( const WitnessID& item : _ineligible_no_turn ) @@ -269,16 +270,16 @@ class generic_witness_scheduler schedule_set.insert( item ); } - flat_set< WitnessID > insertion_set; - insertion_set.reserve( revised_set.size() ); + set< WitnessID > insertion_set; + //insertion_set.reserve( revised_set.size() ); for( const WitnessID& item : revised_set ) { if( current_set.find( item ) == current_set.end() ) insertion_set.insert( item ); } - flat_set< WitnessID > removal_set; - removal_set.reserve( current_set.size() ); + set< WitnessID > removal_set; + //removal_set.reserve( current_set.size() ); for( const WitnessID& item : current_set ) { if( revised_set.find( item ) == revised_set.end() ) @@ -366,7 +367,7 @@ class generic_witness_scheduler std::deque < WitnessID > _schedule; // in _schedule, but not to be replaced - flat_set< WitnessID > _lame_duck; + set< WitnessID > _lame_duck; }; template< typename WitnessID, typename RNG, typename CountType, typename OffsetType, bool debug = true > diff --git a/libraries/db/include/graphene/db/undo_database.hpp b/libraries/db/include/graphene/db/undo_database.hpp index 3b698798..4cf206ee 100644 --- a/libraries/db/include/graphene/db/undo_database.hpp +++ b/libraries/db/include/graphene/db/undo_database.hpp @@ -85,6 +85,7 @@ namespace graphene { namespace db { void disable(); void enable(); + bool enabled()const { return !_disabled; } session start_undo_session(); /** diff --git a/libraries/net/include/graphene/net/message_oriented_connection.hpp b/libraries/net/include/graphene/net/message_oriented_connection.hpp index dec5e994..9547fcef 100644 --- a/libraries/net/include/graphene/net/message_oriented_connection.hpp +++ b/libraries/net/include/graphene/net/message_oriented_connection.hpp @@ -36,26 +36,27 @@ namespace graphene { namespace net { /** uses a secure socket to create a connection that reads and writes a stream of `fc::net::message` objects */ class message_oriented_connection { - public: - message_oriented_connection(message_oriented_connection_delegate* delegate = nullptr); - ~message_oriented_connection(); - fc::tcp_socket& get_socket(); - void accept(); - void bind(const fc::ip::endpoint& local_endpoint); - void connect_to(const fc::ip::endpoint& remote_endpoint); + public: + message_oriented_connection(message_oriented_connection_delegate* delegate = nullptr); + ~message_oriented_connection(); + fc::tcp_socket& get_socket(); - void send_message(const message& message_to_send); - void close_connection(); - void destroy_connection(); + void accept(); + void bind(const fc::ip::endpoint& local_endpoint); + void connect_to(const fc::ip::endpoint& remote_endpoint); - uint64_t get_total_bytes_sent() const; - uint64_t get_total_bytes_received() const; - fc::time_point get_last_message_sent_time() const; - fc::time_point get_last_message_received_time() const; - fc::time_point get_connection_time() const; - fc::sha512 get_shared_secret() const; - private: - std::unique_ptr my; + void send_message(const message& message_to_send); + void close_connection(); + void destroy_connection(); + + uint64_t get_total_bytes_sent() const; + uint64_t get_total_bytes_received() const; + fc::time_point get_last_message_sent_time() const; + fc::time_point get_last_message_received_time() const; + fc::time_point get_connection_time() const; + fc::sha512 get_shared_secret() const; + private: + std::unique_ptr my; }; typedef std::shared_ptr message_oriented_connection_ptr;