From d64c9154a763518556ee84faf74beb9870856a69 Mon Sep 17 00:00:00 2001 From: Nathan Hourt Date: Wed, 8 Jul 2015 17:39:28 -0400 Subject: [PATCH] Progress #17: Lazy load genesis state This speeds up startup for witness_node when starting on a database which is already initialized. --- libraries/app/application.cpp | 50 ++++++++++--------- libraries/chain/db_init.cpp | 2 +- libraries/chain/db_management.cpp | 29 ++--------- .../include/graphene/chain/account_object.hpp | 4 +- .../chain/include/graphene/chain/database.hpp | 34 ++++++++++++- tests/common/database_fixture.cpp | 2 +- tests/tests/block_tests.cpp | 38 +++++++------- tests/tests/operation_tests2.cpp | 2 +- 8 files changed, 86 insertions(+), 75 deletions(-) diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index 8fbebb5e..37566d5d 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -183,12 +183,14 @@ namespace detail { bool clean = !fc::exists(_data_dir / "blockchain/dblock"); fc::create_directories(_data_dir / "blockchain/dblock"); - genesis_state_type initial_state; - if( _options->count("genesis-json") ) - initial_state = fc::json::from_file(_options->at("genesis-json").as()) - .as(); - else - initial_state = create_example_genesis(); + auto initial_state = [&] { + ilog("Initializing database..."); + if( _options->count("genesis-json") ) + return fc::json::from_file(_options->at("genesis-json").as()) + .as(); + else + return create_example_genesis(); + }; if( _options->count("resync-blockchain") ) _chain_db->wipe(_data_dir / "blockchain", true); @@ -196,12 +198,12 @@ namespace detail { if( _options->count("replay-blockchain") ) { ilog("Replaying blockchain on user request."); - _chain_db->reindex(_data_dir/"blockchain", initial_state); + _chain_db->reindex(_data_dir/"blockchain", initial_state()); } else if( clean ) _chain_db->open(_data_dir / "blockchain", initial_state); else { wlog("Detected unclean shutdown. Replaying blockchain..."); - _chain_db->reindex(_data_dir / "blockchain", initial_state); + _chain_db->reindex(_data_dir / "blockchain", initial_state()); } if( _options->count("apiaccess") ) @@ -226,13 +228,13 @@ namespace detail { reset_websocket_tls_server(); } FC_CAPTURE_AND_RETHROW() } - optional< api_access_info > get_api_access_info( const string& username )const + optional< api_access_info > get_api_access_info(const string& username)const { optional< api_access_info > result; - auto it = _apiaccess.permission_map.find( username ); + auto it = _apiaccess.permission_map.find(username); if( it == _apiaccess.permission_map.end() ) { - it = _apiaccess.permission_map.find( "*" ); + it = _apiaccess.permission_map.find("*"); if( it == _apiaccess.permission_map.end() ) return result; } @@ -242,7 +244,7 @@ namespace detail { /** * If delegate has the item, the network has no need to fetch it. */ - virtual bool has_item( const net::item_id& id ) override + virtual bool has_item(const net::item_id& id) override { try { @@ -262,7 +264,7 @@ namespace detail { * * @throws exception if error validating the item, otherwise the item is safe to broadcast on. */ - virtual bool handle_block( const graphene::net::block_message& blk_msg, bool sync_mode ) override + virtual bool handle_block(const graphene::net::block_message& blk_msg, bool sync_mode) override { try { ilog("Got block #${n} from network", ("n", blk_msg.block.block_num())); try { @@ -273,7 +275,7 @@ namespace detail { } } FC_CAPTURE_AND_RETHROW( (blk_msg)(sync_mode) ) } - virtual bool handle_transaction( const graphene::net::trx_message& trx_msg, bool sync_mode ) override + virtual bool handle_transaction(const graphene::net::trx_message& trx_msg, bool sync_mode) override { try { ilog("Got transaction from network"); _chain_db->push_transaction( trx_msg.trx ); @@ -328,18 +330,18 @@ namespace detail { /** * Given the hash of the requested data, fetch the body. */ - virtual message get_item( const item_id& id ) override + virtual message get_item(const item_id& id) override { try { ilog("Request for item ${id}", ("id", id)); if( id.item_type == graphene::net::block_message_type ) { - auto opt_block = _chain_db->fetch_block_by_id( id.item_hash ); + auto opt_block = _chain_db->fetch_block_by_id(id.item_hash); if( !opt_block ) elog("Couldn't find block ${id} -- corresponding ID in our chain is ${id2}", ("id", id.item_hash)("id2", _chain_db->get_block_id_for_num(block_header::num_from_id(id.item_hash)))); FC_ASSERT( opt_block.valid() ); ilog("Serving up block #${num}", ("num", opt_block->block_num())); - return block_message( std::move(*opt_block) ); + return block_message(std::move(*opt_block)); } return trx_message( _chain_db->get_recent_transaction( id.item_hash ) ); } FC_CAPTURE_AND_RETHROW( (id) ) } @@ -364,18 +366,18 @@ namespace detail { * &c. * the last item in the list will be the hash of the most recent block on our preferred chain */ - virtual std::vector get_blockchain_synopsis( uint32_t item_type, - const graphene::net::item_hash_t& reference_point, - uint32_t number_of_blocks_after_reference_point ) override + virtual std::vector get_blockchain_synopsis(uint32_t item_type, + const graphene::net::item_hash_t& reference_point, + uint32_t number_of_blocks_after_reference_point) override { try { std::vector result; result.reserve(30); auto head_block_num = _chain_db->head_block_num(); - result.push_back( _chain_db->head_block_id() ); + result.push_back(_chain_db->head_block_id()); auto current = 1; while( current < head_block_num ) { - result.push_back( _chain_db->get_block_id_for_num( head_block_num - current ) ); + result.push_back(_chain_db->get_block_id_for_num(head_block_num - current)); current = current*2; } std::reverse( result.begin(), result.end() ); @@ -390,7 +392,7 @@ namespace detail { * @param item_count the number of items known to the node that haven't been sent to handle_item() yet. * After `item_count` more calls to handle_item(), the node will be in sync */ - virtual void sync_status( uint32_t item_type, uint32_t item_count ) override + virtual void sync_status(uint32_t item_type, uint32_t item_count) override { // any status reports to GUI go here } @@ -398,7 +400,7 @@ namespace detail { /** * Call any time the number of connected peers changes. */ - virtual void connection_count_changed( uint32_t c ) override + virtual void connection_count_changed(uint32_t c) override { // any status reports to GUI go here } diff --git a/libraries/chain/db_init.cpp b/libraries/chain/db_init.cpp index 52027015..3101c9d0 100644 --- a/libraries/chain/db_init.cpp +++ b/libraries/chain/db_init.cpp @@ -248,7 +248,7 @@ void database::init_genesis(const genesis_state_type& genesis_state) p.parameters.current_fees.set_all_fees(0); }); - create( [&](dynamic_global_property_object& p) { + create([&](dynamic_global_property_object& p) { p.time = genesis_state.initial_timestamp; p.witness_budget = 0; }); diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp index 14dbba9d..0dc41e4d 100644 --- a/libraries/chain/db_management.cpp +++ b/libraries/chain/db_management.cpp @@ -20,6 +20,8 @@ #include +#include + namespace graphene { namespace chain { database::database() @@ -33,31 +35,10 @@ database::~database(){ _pending_block_session->commit(); } -void database::open( const fc::path& data_dir, const genesis_state_type& initial_allocation ) -{ try { - object_database::open( data_dir ); - - _block_id_to_block.open(data_dir / "database" / "block_num_to_block"); - - if( !find(global_property_id_type()) ) - { -// ilog( "Init Genesis State" ); - init_genesis(initial_allocation); - } - - _pending_block.previous = head_block_id(); - _pending_block.timestamp = head_block_time(); - - auto last_block= _block_id_to_block.last(); - if( last_block.valid() ) - _fork_db.start_block( *last_block ); - -} FC_CAPTURE_AND_RETHROW( (data_dir) ) } - void database::reindex(fc::path data_dir, const genesis_state_type& initial_allocation) { try { wipe(data_dir, false); - open(data_dir, initial_allocation); + open(data_dir, [&initial_allocation]{return initial_allocation;}); auto start = fc::time_point::now(); auto last_block = _block_id_to_block.last(); @@ -69,13 +50,13 @@ void database::reindex(fc::path data_dir, const genesis_state_type& initial_allo //_undo_db.disable(); for( uint32_t i = 1; i <= last_block_num; ++i ) { - apply_block( *_block_id_to_block.fetch_by_number(i), skip_delegate_signature | + apply_block(*_block_id_to_block.fetch_by_number(i), skip_delegate_signature | skip_transaction_signatures | skip_undo_block | skip_undo_transaction | skip_transaction_dupe_check | skip_tapos_check | - skip_authority_check ); + skip_authority_check); } //_undo_db.enable(); auto end = fc::time_point::now(); diff --git a/libraries/chain/include/graphene/chain/account_object.hpp b/libraries/chain/include/graphene/chain/account_object.hpp index 0e313160..b8ed6712 100644 --- a/libraries/chain/include/graphene/chain/account_object.hpp +++ b/libraries/chain/include/graphene/chain/account_object.hpp @@ -161,7 +161,7 @@ class database; /// The memo key is the key this account will typically use to encrypt/sign transaction memos and other non- /// validated account activities. This field is here to prevent confusion if the active authority has zero or /// multiple keys in it. - public_key_type memo_key; + public_key_type memo_key; /// If this field is set to an account ID other than 0, this account's votes will be ignored and its stake /// will be counted as voting for the referenced account's selected votes instead. account_id_type voting_account; @@ -257,7 +257,7 @@ class database; /** * @brief This secondary index will allow a reverse lookup of all accounts that a particular key or account - * is an potential signing authority. + * is an potential signing authority. */ class account_member_index : public secondary_index { diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp index 0f06eb89..6d791070 100644 --- a/libraries/chain/include/graphene/chain/database.hpp +++ b/libraries/chain/include/graphene/chain/database.hpp @@ -93,7 +93,19 @@ namespace graphene { namespace chain { skip_assert_evaluation = 0x400 ///< used while reindexing }; - void open(const fc::path& data_dir, const genesis_state_type& initial_allocation = genesis_state_type()); + /** + * @brief Open a database, creating a new one if necessary + * + * Opens a database in the specified directory. If no initialized database is found, genesis_loader is called + * and its return value is used as the genesis state when initializing the new database + * + * genesis_loader will not be called if an existing database is found. + * + * @param data_dir Path to open or create database in + * @param genesis_loader A callable object which returns the genesis state to initialize new databases on + */ + template + void open(const fc::path& data_dir, F&& genesis_loader); /** * @brief Rebuild object graph from block history and open detabase * @@ -477,4 +489,24 @@ namespace graphene { namespace chain { (void)l; } } + + template + void database::open(const fc::path& data_dir, F&& genesis_loader) + { try { + object_database::open(data_dir); + + _block_id_to_block.open(data_dir / "database" / "block_num_to_block"); + + if( !find(global_property_id_type()) ) + init_genesis(genesis_loader()); + + _pending_block.previous = head_block_id(); + _pending_block.timestamp = head_block_time(); + + auto last_block= _block_id_to_block.last(); + if( last_block.valid() ) + _fork_db.start_block( *last_block ); + + } FC_CAPTURE_AND_RETHROW( (data_dir) ) } + } } diff --git a/tests/common/database_fixture.cpp b/tests/common/database_fixture.cpp index c2206243..7c7e69ab 100644 --- a/tests/common/database_fixture.cpp +++ b/tests/common/database_fixture.cpp @@ -286,7 +286,7 @@ void database_fixture::open_database() { if( !data_dir ) { data_dir = fc::temp_directory(); - db.open(data_dir->path(), genesis_state); + db.open(data_dir->path(), [this]{return genesis_state;}); } } diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp index 69aae949..279cbbc0 100644 --- a/tests/tests/block_tests.cpp +++ b/tests/tests/block_tests.cpp @@ -129,7 +129,7 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) auto delegate_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); { database db; - db.open(data_dir.path(), make_genesis() ); + db.open(data_dir.path(), make_genesis ); b = db.generate_block(now, db.get_scheduled_witness(1).first, delegate_priv_key, database::skip_nothing); for( uint32_t i = 1; i < 200; ++i ) @@ -137,25 +137,25 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) BOOST_CHECK( db.head_block_id() == b.id() ); witness_id_type prev_witness = b.witness; now += db.block_interval(); - witness_id_type cur_witness = db.get_scheduled_witness( 1 ).first; + witness_id_type cur_witness = db.get_scheduled_witness(1).first; BOOST_CHECK( cur_witness != prev_witness ); - b = db.generate_block( now, cur_witness, delegate_priv_key, database::skip_nothing ); + b = db.generate_block(now, cur_witness, delegate_priv_key, database::skip_nothing); BOOST_CHECK( b.witness == cur_witness ); } db.close(); } { database db; - db.open(data_dir.path() ); + db.open(data_dir.path(), []{return genesis_state_type();}); BOOST_CHECK_EQUAL( db.head_block_num(), 200 ); for( uint32_t i = 0; i < 200; ++i ) { BOOST_CHECK( db.head_block_id() == b.id() ); witness_id_type prev_witness = b.witness; now += db.block_interval(); - witness_id_type cur_witness = db.get_scheduled_witness( 1 ).first; + witness_id_type cur_witness = db.get_scheduled_witness(1).first; BOOST_CHECK( cur_witness != prev_witness ); - b = db.generate_block( now, cur_witness, delegate_priv_key, database::skip_nothing ); + b = db.generate_block(now, cur_witness, delegate_priv_key, database::skip_nothing); } BOOST_CHECK_EQUAL( db.head_block_num(), 400 ); } @@ -171,7 +171,7 @@ BOOST_AUTO_TEST_CASE( undo_block ) fc::temp_directory data_dir; { database db; - db.open(data_dir.path(), make_genesis() ); + db.open(data_dir.path(), make_genesis); fc::time_point_sec now( GRAPHENE_TESTING_GENESIS_TIMESTAMP ); auto delegate_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); @@ -211,9 +211,9 @@ BOOST_AUTO_TEST_CASE( fork_blocks ) fc::time_point_sec now( GRAPHENE_TESTING_GENESIS_TIMESTAMP ); database db1; - db1.open(data_dir1.path(), make_genesis()); + db1.open(data_dir1.path(), make_genesis); database db2; - db2.open(data_dir2.path(), make_genesis()); + db2.open(data_dir2.path(), make_genesis); auto delegate_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); for( uint32_t i = 0; i < 10; ++i ) @@ -276,7 +276,7 @@ BOOST_AUTO_TEST_CASE( undo_pending ) fc::temp_directory data_dir; { database db; - db.open(data_dir.path(), make_genesis()); + db.open(data_dir.path(), make_genesis); auto delegate_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); public_key_type delegate_pub_key = delegate_priv_key.get_public_key(); @@ -334,8 +334,8 @@ BOOST_AUTO_TEST_CASE( switch_forks_undo_create ) dir2; database db1, db2; - db1.open(dir1.path(), make_genesis()); - db2.open(dir2.path(), make_genesis()); + db1.open(dir1.path(), make_genesis); + db2.open(dir2.path(), make_genesis); fc::time_point_sec now( GRAPHENE_TESTING_GENESIS_TIMESTAMP ); auto delegate_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); @@ -392,8 +392,8 @@ BOOST_AUTO_TEST_CASE( duplicate_transactions ) dir2; database db1, db2; - db1.open(dir1.path(), make_genesis()); - db2.open(dir2.path(), make_genesis()); + db1.open(dir1.path(), make_genesis); + db2.open(dir2.path(), make_genesis); auto skip_sigs = database::skip_transaction_signatures | database::skip_authority_check; @@ -441,8 +441,8 @@ BOOST_AUTO_TEST_CASE( tapos ) dir2; database db1, db2; - db1.open(dir1.path(), make_genesis()); - db2.open(dir2.path(), make_genesis()); + db1.open(dir1.path(), make_genesis); + db2.open(dir2.path(), make_genesis); const account_object& init1 = *db1.get_index_type().indices().get().find("init1"); @@ -451,7 +451,7 @@ BOOST_AUTO_TEST_CASE( tapos ) const graphene::db::index& account_idx = db1.get_index(protocol_ids, account_object_type); now += db1.block_interval(); - auto b = db1.generate_block( now, db1.get_scheduled_witness( 1 ).first, delegate_priv_key, database::skip_nothing ); + auto b = db1.generate_block(now, db1.get_scheduled_witness( 1 ).first, delegate_priv_key, database::skip_nothing); signed_transaction trx; //This transaction must be in the next block after its reference, or it is invalid. @@ -467,10 +467,6 @@ BOOST_AUTO_TEST_CASE( tapos ) db1.push_transaction(trx); now += db1.block_interval(); b = db1.generate_block(now, db1.get_scheduled_witness(1).first, delegate_priv_key, database::skip_nothing); - /* - now += db1.block_interval(); - b = db1.generate_block(now, db1.get_scheduled_witness(1).first, delegate_priv_key, database::skip_nothing); - */ trx.clear(); trx.operations.push_back(transfer_operation({asset(), account_id_type(), nathan_id, asset(50)})); diff --git a/tests/tests/operation_tests2.cpp b/tests/tests/operation_tests2.cpp index da81c72d..9d25826f 100644 --- a/tests/tests/operation_tests2.cpp +++ b/tests/tests/operation_tests2.cpp @@ -1001,7 +1001,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test ) genesis_state.initial_accounts.emplace_back("n", n_key.get_public_key()); - db.open(td.path(), genesis_state); + db.open(td.path(), [this]{return genesis_state;}); const balance_object& balance = balance_id_type()(db); BOOST_CHECK_EQUAL(balance.balance.amount.value, 1); BOOST_CHECK_EQUAL(balance_id_type(1)(db).balance.amount.value, 1);