From 17417037c648fd01529b3e894ff3377833cc3b1d Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Wed, 12 Jul 2017 22:03:57 +0200 Subject: [PATCH] Moved reindex logic into database / chain_database, make use of additional blocks in block_database Fixed tests wrt db.open --- libraries/app/application.cpp | 67 +++--------------- libraries/chain/db_management.cpp | 69 +++++++++++-------- .../chain/include/graphene/chain/database.hpp | 6 +- libraries/db/object_database.cpp | 7 ++ tests/benchmarks/genesis_allocation.cpp | 6 +- tests/common/database_fixture.cpp | 2 +- tests/generate_empty_blocks/main.cpp | 2 +- tests/tests/block_tests.cpp | 26 +++---- tests/tests/operation_tests2.cpp | 2 +- 9 files changed, 81 insertions(+), 106 deletions(-) diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index 5e4f9c7e..1a0b8b65 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -300,7 +300,6 @@ namespace detail { ~application_impl() { - fc::remove_all(_data_dir / "blockchain/dblock"); } void set_dbg_init_key( genesis_state_type& genesis, const std::string& init_key ) @@ -314,8 +313,7 @@ namespace detail { void startup() { try { - bool clean = !fc::exists(_data_dir / "blockchain/dblock"); - fc::create_directories(_data_dir / "blockchain/dblock"); + fc::create_directories(_data_dir / "blockchain"); auto initial_state = [&] { ilog("Initializing database..."); @@ -381,64 +379,17 @@ namespace detail { bool replay = false; std::string replay_reason = "reason not provided"; - // never replay if data dir is empty - if( fc::exists( _data_dir ) && fc::directory_iterator( _data_dir ) != fc::directory_iterator() ) - { - if( _options->count("replay-blockchain") ) - { - replay = true; - replay_reason = "replay-blockchain argument specified"; - } - else if( !clean ) - { - replay = true; - replay_reason = "unclean shutdown detected"; - } - else if( !fc::exists( _data_dir / "db_version" ) ) - { - replay = true; - replay_reason = "db_version file not found"; - } - else - { - std::string version_string; - fc::read_file_contents( _data_dir / "db_version", version_string ); + if( _options->count("replay-blockchain") ) + _chain_db->wipe( _data_dir / "blockchain", false ); - if( version_string != GRAPHENE_CURRENT_DB_VERSION ) - { - replay = true; - replay_reason = "db_version file content mismatch"; - } - } + try + { + _chain_db->open( _data_dir / "blockchain", initial_state, GRAPHENE_CURRENT_DB_VERSION ); } - - if( !replay ) + catch( const fc::exception& e ) { - try - { - _chain_db->open( _data_dir / "blockchain", initial_state ); - } - catch( const fc::exception& e ) - { - ilog( "Caught exception ${e} in open()", ("e", e.to_detail_string()) ); - - replay = true; - replay_reason = "exception in open()"; - } - } - - if( replay ) - { - ilog( "Replaying blockchain due to: ${reason}", ("reason", replay_reason) ); - - fc::remove_all( _data_dir / "db_version" ); - _chain_db->reindex( _data_dir / "blockchain", initial_state() ); - - const auto mode = std::ios::out | std::ios::binary | std::ios::trunc; - std::ofstream db_version( (_data_dir / "db_version").generic_string().c_str(), mode ); - std::string version_string = GRAPHENE_CURRENT_DB_VERSION; - db_version.write( version_string.c_str(), version_string.size() ); - db_version.close(); + elog( "Caught exception ${e} in open(), you might want to force a replay", ("e", e.to_detail_string()) ); + throw; } if( _options->count("force-validate") ) diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp index 68f6fad1..ca4004e6 100644 --- a/libraries/chain/db_management.cpp +++ b/libraries/chain/db_management.cpp @@ -47,33 +47,39 @@ database::~database() clear_pending(); } -void database::reindex(fc::path data_dir, const genesis_state_type& initial_allocation) +void database::reindex( fc::path data_dir ) { try { - ilog( "reindexing blockchain" ); - wipe(data_dir, false); - open(data_dir, [&initial_allocation]{return initial_allocation;}); - - auto start = fc::time_point::now(); auto last_block = _block_id_to_block.last(); if( !last_block ) { elog( "!no last block" ); edump((last_block)); return; } + if( last_block->block_num() <= head_block_num()) return; + ilog( "reindexing blockchain" ); + auto start = fc::time_point::now(); const auto last_block_num = last_block->block_num(); + uint32_t flush_point = last_block_num - 10000; + uint32_t undo_point = last_block_num - 50; ilog( "Replaying blocks..." ); - // Right now, we leave undo_db enabled when replaying when the bookie plugin is + // Right now, we leave undo_db enabled when replaying when the bookie plugin is // enabled. It depends on new/changed/removed object notifications, and those are // only fired when the undo_db is enabled if (!_slow_replays) _undo_db.disable(); - for( uint32_t i = 1; i <= last_block_num; ++i ) + for( uint32_t i = head_block_num() + 1; i <= last_block_num; ++i ) { - if( i == 1 || - i % 10000 == 0 ) - std::cerr << " " << double(i*100)/last_block_num << "% "<< i << " of " < block = _block_id_to_block.fetch_by_number(i); if( !block.valid() ) { @@ -127,10 +133,29 @@ void database::wipe(const fc::path& data_dir, bool include_blocks) void database::open( const fc::path& data_dir, - std::function genesis_loader) + std::function genesis_loader, + const std::string& db_version) { try { + bool wipe_object_db = false; + if( !fc::exists( data_dir / "db_version" ) ) + wipe_object_db = true; + else + { + std::string version_string; + fc::read_file_contents( data_dir / "db_version", version_string ); + wipe_object_db = ( version_string != db_version ); + } + if( wipe_object_db ) { + ilog("Wiping object_database due to missing or wrong version"); + object_database::wipe( data_dir ); + std::ofstream version_file( (data_dir / "db_version").generic_string().c_str(), + std::ios::out | std::ios::binary | std::ios::trunc ); + version_file.write( db_version.c_str(), db_version.size() ); + version_file.close(); + } + object_database::open(data_dir); _block_id_to_block.open(data_dir / "database" / "block_num_to_block"); @@ -138,15 +163,13 @@ void database::open( if( !find(global_property_id_type()) ) init_genesis(genesis_loader()); - fc::optional last_block = _block_id_to_block.last(); + fc::optional last_block = _block_id_to_block.last_id(); if( last_block.valid() ) { - _fork_db.start_block( *last_block ); - if( last_block->id() != head_block_id() ) - { - FC_ASSERT( head_block_num() == 0, "last block ID does not match current chain state", - ("last_block->id", last_block->id())("head_block_num",head_block_num()) ); - } + FC_ASSERT( *last_block >= head_block_id(), + "last block ID does not match current chain state", + ("last_block->id", last_block)("head_block_id",head_block_num()) ); + reindex( data_dir ); } } FC_CAPTURE_LOG_AND_RETHROW( (data_dir) ) @@ -167,17 +190,9 @@ void database::close(bool rewind) while( head_block_num() > cutoff ) { - // elog("pop"); block_id_type popped_block_id = head_block_id(); pop_block(); _fork_db.remove(popped_block_id); // doesn't throw on missing - try - { - _block_id_to_block.remove(popped_block_id); - } - catch (const fc::key_not_found_exception&) - { - } } } catch ( const fc::exception& e ) diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp index 0c6dcb0f..77699ee4 100644 --- a/libraries/chain/include/graphene/chain/database.hpp +++ b/libraries/chain/include/graphene/chain/database.hpp @@ -91,10 +91,12 @@ namespace graphene { namespace chain { * * @param data_dir Path to open or create database in * @param genesis_loader A callable object which returns the genesis state to initialize new databases on + * @param db_version a version string that changes when the internal database format and/or logic is modified */ void open( const fc::path& data_dir, - std::function genesis_loader ); + std::function genesis_loader, + const std::string& db_version ); /** * @brief Rebuild object graph from block history and open detabase @@ -102,7 +104,7 @@ namespace graphene { namespace chain { * This method may be called after or instead of @ref database::open, and will rebuild the object graph by * replaying blockchain history. When this method exits successfully, the database will be open. */ - void reindex(fc::path data_dir, const genesis_state_type& initial_allocation = genesis_state_type()); + void reindex(fc::path data_dir); /** * @brief wipe Delete database from disk, and potentially the raw chain as well. diff --git a/libraries/db/object_database.cpp b/libraries/db/object_database.cpp index 29d83ae7..6e1fdea2 100644 --- a/libraries/db/object_database.cpp +++ b/libraries/db/object_database.cpp @@ -71,6 +71,7 @@ index& object_database::get_mutable_index(uint8_t space_id, uint8_t type_id) void object_database::flush() { // ilog("Save object_database in ${d}", ("d", _data_dir)); + fc::create_directories( _data_dir / "object_database" / "lock" ); for( uint32_t space = 0; space < _index.size(); ++space ) { fc::create_directories( _data_dir / "object_database" / fc::to_string(space) ); @@ -79,6 +80,7 @@ void object_database::flush() if( _index[space][type] ) _index[space][type]->save( _data_dir / "object_database" / fc::to_string(space)/fc::to_string(type) ); } + fc::remove_all( _data_dir / "object_database" / "lock" ); } void object_database::wipe(const fc::path& data_dir) @@ -91,6 +93,11 @@ void object_database::wipe(const fc::path& data_dir) void object_database::open(const fc::path& data_dir) { try { + if( fc::exists( _data_dir / "object_database" / "lock" ) ) + { + wlog("Ignoring locked object_database"); + return; + } ilog("Opening object database from ${d} ...", ("d", data_dir)); _data_dir = data_dir; for( uint32_t space = 0; space < _index.size(); ++space ) diff --git a/tests/benchmarks/genesis_allocation.cpp b/tests/benchmarks/genesis_allocation.cpp index 61a3b1b8..a17a16fa 100644 --- a/tests/benchmarks/genesis_allocation.cpp +++ b/tests/benchmarks/genesis_allocation.cpp @@ -68,7 +68,7 @@ BOOST_AUTO_TEST_CASE( genesis_and_persistence_bench ) { database db; - db.open(data_dir.path(), [&]{return genesis_state;}); + db.open(data_dir.path(), [&]{return genesis_state;}, "test"); for( int i = 11; i < account_count + 11; ++i) BOOST_CHECK(db.get_balance(account_id_type(i), asset_id_type()).amount == GRAPHENE_MAX_SHARE_SUPPLY / account_count); @@ -81,7 +81,7 @@ BOOST_AUTO_TEST_CASE( genesis_and_persistence_bench ) database db; fc::time_point start_time = fc::time_point::now(); - db.open(data_dir.path(), [&]{return genesis_state;}); + db.open(data_dir.path(), [&]{return genesis_state;}, "test"); ilog("Opened database in ${t} milliseconds.", ("t", (fc::time_point::now() - start_time).count() / 1000)); for( int i = 11; i < account_count + 11; ++i) @@ -116,7 +116,7 @@ BOOST_AUTO_TEST_CASE( genesis_and_persistence_bench ) auto start_time = fc::time_point::now(); wlog( "about to start reindex..." ); - db.reindex(data_dir.path(), genesis_state); + db.open(data_dir.path(), [&]{return genesis_state;}, "force_wipe"); ilog("Replayed database in ${t} milliseconds.", ("t", (fc::time_point::now() - start_time).count() / 1000)); for( int i = 0; i < blocks_to_produce; ++i ) diff --git a/tests/common/database_fixture.cpp b/tests/common/database_fixture.cpp index e6a0b327..1741c787 100644 --- a/tests/common/database_fixture.cpp +++ b/tests/common/database_fixture.cpp @@ -355,7 +355,7 @@ void database_fixture::open_database() { if( !data_dir ) { data_dir = fc::temp_directory( graphene::utilities::temp_directory_path() ); - db.open(data_dir->path(), [this]{return genesis_state;}); + db.open(data_dir->path(), [this]{return genesis_state;}, "test"); } } diff --git a/tests/generate_empty_blocks/main.cpp b/tests/generate_empty_blocks/main.cpp index 1b45340d..b6a2ca95 100644 --- a/tests/generate_empty_blocks/main.cpp +++ b/tests/generate_empty_blocks/main.cpp @@ -124,7 +124,7 @@ int main( int argc, char** argv ) database db; fc::path db_path = data_dir / "db"; - db.open(db_path, [&]() { return genesis; } ); + db.open(db_path, [&]() { return genesis; }, "TEST" ); uint32_t slot = 1; uint32_t missed = 0; diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp index 07609d4b..5990b12b 100644 --- a/tests/tests/block_tests.cpp +++ b/tests/tests/block_tests.cpp @@ -138,7 +138,7 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) signed_block cutoff_block; { database db; - db.open(data_dir.path(), make_genesis ); + db.open(data_dir.path(), make_genesis, "TEST" ); b = db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); // TODO: Change this test when we correct #406 @@ -162,7 +162,7 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) } { database db; - db.open(data_dir.path(), []{return genesis_state_type();}); + db.open(data_dir.path(), []{return genesis_state_type();}, "TEST"); BOOST_CHECK_EQUAL( db.head_block_num(), cutoff_block.block_num() ); b = cutoff_block; for( uint32_t i = 0; i < 200; ++i ) @@ -187,7 +187,7 @@ BOOST_AUTO_TEST_CASE( undo_block ) fc::temp_directory data_dir( graphene::utilities::temp_directory_path() ); { database db; - db.open(data_dir.path(), make_genesis); + db.open(data_dir.path(), make_genesis, "TEST"); fc::time_point_sec now( GRAPHENE_TESTING_GENESIS_TIMESTAMP ); std::vector< time_point_sec > time_stack; @@ -236,9 +236,9 @@ BOOST_AUTO_TEST_CASE( fork_blocks ) fc::temp_directory data_dir2( graphene::utilities::temp_directory_path() ); database db1; - db1.open(data_dir1.path(), make_genesis); + db1.open(data_dir1.path(), make_genesis, "TEST"); database db2; - db2.open(data_dir2.path(), make_genesis); + db2.open(data_dir2.path(), make_genesis, "TEST"); BOOST_CHECK( db1.get_chain_id() == db2.get_chain_id() ); auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); @@ -381,7 +381,7 @@ BOOST_AUTO_TEST_CASE( undo_pending ) fc::temp_directory data_dir( graphene::utilities::temp_directory_path() ); { database db; - db.open(data_dir.path(), make_genesis); + db.open(data_dir.path(), make_genesis, "TEST"); auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); public_key_type init_account_pub_key = init_account_priv_key.get_public_key(); @@ -446,8 +446,8 @@ BOOST_AUTO_TEST_CASE( switch_forks_undo_create ) dir2( graphene::utilities::temp_directory_path() ); database db1, db2; - db1.open(dir1.path(), make_genesis); - db2.open(dir2.path(), make_genesis); + db1.open(dir1.path(), make_genesis, "TEST"); + db2.open(dir2.path(), make_genesis, "TEST"); BOOST_CHECK( db1.get_chain_id() == db2.get_chain_id() ); auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); @@ -505,8 +505,8 @@ BOOST_AUTO_TEST_CASE( duplicate_transactions ) dir2( graphene::utilities::temp_directory_path() ); database db1, db2; - db1.open(dir1.path(), make_genesis); - db2.open(dir2.path(), make_genesis); + db1.open(dir1.path(), make_genesis, "TEST"); + db2.open(dir2.path(), make_genesis, "TEST"); BOOST_CHECK( db1.get_chain_id() == db2.get_chain_id() ); auto skip_sigs = database::skip_transaction_signatures | database::skip_authority_check; @@ -555,7 +555,7 @@ BOOST_AUTO_TEST_CASE( tapos ) try { fc::temp_directory dir1( graphene::utilities::temp_directory_path() ); database db1; - db1.open(dir1.path(), make_genesis); + db1.open(dir1.path(), make_genesis, "TEST"); const account_object& init1 = *db1.get_index_type().indices().get().find("init1"); @@ -1106,7 +1106,7 @@ BOOST_FIXTURE_TEST_CASE( transaction_invalidated_in_cache, database_fixture ) fc::temp_directory data_dir2( graphene::utilities::temp_directory_path() ); database db2; - db2.open(data_dir2.path(), make_genesis); + db2.open(data_dir2.path(), make_genesis, "TEST"); BOOST_CHECK( db.get_chain_id() == db2.get_chain_id() ); while( db2.head_block_num() < db.head_block_num() ) @@ -1269,7 +1269,7 @@ BOOST_AUTO_TEST_CASE( genesis_reserve_ids ) genesis_state.initial_assets.push_back( usd ); return genesis_state; - } ); + }, "TEST" ); const auto& acct_idx = db.get_index_type().indices().get(); auto acct_itr = acct_idx.find("init0"); diff --git a/tests/tests/operation_tests2.cpp b/tests/tests/operation_tests2.cpp index 98d40207..3981270d 100644 --- a/tests/tests/operation_tests2.cpp +++ b/tests/tests/operation_tests2.cpp @@ -1111,7 +1111,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test ) auto _sign = [&]( signed_transaction& tx, const private_key_type& key ) { tx.sign( key, db.get_chain_id() ); }; - db.open(td.path(), [this]{return genesis_state;}); + db.open(td.path(), [this]{return genesis_state;}, "TEST"); const balance_object& balance = balance_id_type()(db); BOOST_CHECK_EQUAL(balance.balance.amount.value, 1); BOOST_CHECK_EQUAL(balance_id_type(1)(db).balance.amount.value, 1);