From b7515084047597881adc7d51182e3c715e72a0f7 Mon Sep 17 00:00:00 2001 From: Ronak Patel Date: Mon, 2 Sep 2019 18:23:19 +0530 Subject: [PATCH 01/16] Merged changes from Bitshares PR 1036 --- .../graphene/chain/proposal_object.hpp | 2 +- libraries/chain/proposal_evaluator.cpp | 17 ++----- libraries/chain/proposal_object.cpp | 2 - tests/tests/authority_tests.cpp | 50 +++++++++++++++++-- 4 files changed, 51 insertions(+), 20 deletions(-) diff --git a/libraries/chain/include/graphene/chain/proposal_object.hpp b/libraries/chain/include/graphene/chain/proposal_object.hpp index d41ea7ea..92a776cd 100644 --- a/libraries/chain/include/graphene/chain/proposal_object.hpp +++ b/libraries/chain/include/graphene/chain/proposal_object.hpp @@ -51,7 +51,7 @@ class proposal_object : public abstract_object flat_set available_owner_approvals; flat_set available_key_approvals; account_id_type proposer; - + std::string fail_reason; bool is_authorized_to_execute(database& db)const; }; diff --git a/libraries/chain/proposal_evaluator.cpp b/libraries/chain/proposal_evaluator.cpp index a6640ea4..3a44ca5c 100644 --- a/libraries/chain/proposal_evaluator.cpp +++ b/libraries/chain/proposal_evaluator.cpp @@ -244,20 +244,6 @@ void_result proposal_update_evaluator::do_evaluate(const proposal_update_operati "", ("id", id)("available", _proposal->available_owner_approvals) ); } - /* All authority checks happen outside of evaluators - if( (d.get_node_properties().skip_flags & database::skip_authority_check) == 0 ) - { - for( const auto& id : o.key_approvals_to_add ) - { - FC_ASSERT( trx_state->signed_by(id) ); - } - for( const auto& id : o.key_approvals_to_remove ) - { - FC_ASSERT( trx_state->signed_by(id) ); - } - } - */ - return void_result(); } FC_CAPTURE_AND_RETHROW( (o) ) } @@ -293,6 +279,9 @@ void_result proposal_update_evaluator::do_apply(const proposal_update_operation& try { _processed_transaction = d.push_proposal(*_proposal); } catch(fc::exception& e) { + d.modify(*_proposal, [&e](proposal_object& p) { + p.fail_reason = e.to_string(fc::log_level(fc::log_level::all)); + }); wlog("Proposed transaction ${id} failed to apply once approved with exception:\n----\n${reason}\n----\nWill try again when it expires.", ("id", o.proposal)("reason", e.to_detail_string())); _proposal_failed = true; diff --git a/libraries/chain/proposal_object.cpp b/libraries/chain/proposal_object.cpp index 565964a5..2313a492 100644 --- a/libraries/chain/proposal_object.cpp +++ b/libraries/chain/proposal_object.cpp @@ -43,8 +43,6 @@ bool proposal_object::is_authorized_to_execute(database& db) const } catch ( const fc::exception& e ) { - //idump((available_active_approvals)); - //wlog((e.to_detail_string())); return false; } return true; diff --git a/tests/tests/authority_tests.cpp b/tests/tests/authority_tests.cpp index f5efbb9d..1f6bc1fb 100644 --- a/tests/tests/authority_tests.cpp +++ b/tests/tests/authority_tests.cpp @@ -389,6 +389,49 @@ BOOST_AUTO_TEST_CASE( proposed_single_account ) } } +BOOST_AUTO_TEST_CASE( proposal_failure ) +{ + try + { + ACTORS( (bob) (alice) ); + + fund( bob, asset(1000000) ); + fund( alice, asset(1000000) ); + + // create proposal that will eventually fail due to lack of funds + transfer_operation top; + top.to = alice_id; + top.from = bob_id; + top.amount = asset(2000000); + proposal_create_operation pop; + pop.proposed_ops.push_back( { top } ); + pop.expiration_time = db.head_block_time() + fc::days(1); + pop.fee_paying_account = bob_id; + trx.operations.push_back( pop ); + trx.signatures.clear(); + sign( trx, bob_private_key ); + processed_transaction processed = PUSH_TX( db, trx ); + proposal_object prop = db.get(processed.operation_results.front().get()); + trx.clear(); + generate_block(); + // add signature + proposal_update_operation up_op; + up_op.proposal = prop.id; + up_op.fee_paying_account = bob_id; + up_op.active_approvals_to_add.emplace( bob_id ); + trx.operations.push_back( up_op ); + sign( trx, bob_private_key ); + PUSH_TX( db, trx ); + trx.clear(); + + // check fail reason + const proposal_object& result = db.get(prop.id); + BOOST_CHECK(!result.fail_reason.empty()); + BOOST_CHECK_EQUAL( result.fail_reason.substr(0, 16), "Assert Exception"); + } + FC_LOG_AND_RETHROW() +} + /// Verify that committee authority cannot be invoked in a normal transaction BOOST_AUTO_TEST_CASE( committee_authority ) { try { @@ -478,9 +521,10 @@ BOOST_AUTO_TEST_CASE( committee_authority ) // Should throw because the transaction is now in review. GRAPHENE_CHECK_THROW(PUSH_TX( db, trx ), fc::exception); - // generate_blocks(prop.expiration_time); - // fails - // BOOST_CHECK_EQUAL(get_balance(nathan, asset_id_type()(db)), 100000); + generate_blocks(prop.expiration_time); + BOOST_CHECK_EQUAL(get_balance(nathan, asset_id_type()(db)), 100000); + // proposal deleted + BOOST_CHECK_THROW( db.get(prop.id), fc::exception ); } FC_LOG_AND_RETHROW() } BOOST_FIXTURE_TEST_CASE( fired_committee_members, database_fixture ) From 42680456b683afd730f8c9ca034cf5d2306e9e8f Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Wed, 12 Jul 2017 21:13:26 +0200 Subject: [PATCH 02/16] Improved resilience of block database against corruption --- libraries/chain/block_database.cpp | 78 ++++++++----------- .../include/graphene/chain/block_database.hpp | 3 + 2 files changed, 35 insertions(+), 46 deletions(-) diff --git a/libraries/chain/block_database.cpp b/libraries/chain/block_database.cpp index 214459f0..9c351624 100644 --- a/libraries/chain/block_database.cpp +++ b/libraries/chain/block_database.cpp @@ -206,34 +206,41 @@ optional block_database::fetch_by_number( uint32_t block_num )cons return optional(); } -optional block_database::last()const -{ +optional block_database::last_index_entry()const { try { index_entry e; + _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); + std::streampos pos = _block_num_to_pos.tellg(); + if( pos < sizeof(index_entry) ) + return optional(); - if( _block_num_to_pos.tellp() < sizeof(index_entry) ) - return optional(); + if( pos % sizeof(index_entry) != 0 ) + pos -= pos % sizeof(index_entry); - _block_num_to_pos.seekg( -sizeof(index_entry), _block_num_to_pos.end ); - _block_num_to_pos.read( (char*)&e, sizeof(e) ); - uint64_t pos = _block_num_to_pos.tellg(); while( e.block_size == 0 && pos > 0 ) { pos -= sizeof(index_entry); _block_num_to_pos.seekg( pos ); _block_num_to_pos.read( (char*)&e, sizeof(e) ); + + if( e.block_size > 0 ) + try + { + vector data( e.block_size ); + _blocks.seekg( e.block_pos ); + _blocks.read( data.data(), e.block_size ); + auto result = fc::raw::unpack(data); + return e; + } + catch (const fc::exception&) + { + } + catch (const std::exception&) + { + } } - - if( e.block_size == 0 ) - return optional(); - - vector data( e.block_size ); - _blocks.seekg( e.block_pos ); - _blocks.read( data.data(), e.block_size ); - auto result = fc::raw::unpack(data); - return result; } catch (const fc::exception&) { @@ -241,42 +248,21 @@ optional block_database::last()const catch (const std::exception&) { } + return optional(); +} + +optional block_database::last()const +{ + optional entry = last_index_entry(); + if( entry.valid() ) return fetch_by_number( block_header::num_from_id(entry->block_id) ); return optional(); } optional block_database::last_id()const { - try - { - index_entry e; - _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); - - if( _block_num_to_pos.tellp() < sizeof(index_entry) ) - return optional(); - - _block_num_to_pos.seekg( -sizeof(index_entry), _block_num_to_pos.end ); - _block_num_to_pos.read( (char*)&e, sizeof(e) ); - uint64_t pos = _block_num_to_pos.tellg(); - while( e.block_size == 0 && pos > 0 ) - { - pos -= sizeof(index_entry); - _block_num_to_pos.seekg( pos ); - _block_num_to_pos.read( (char*)&e, sizeof(e) ); - } - - if( e.block_size == 0 ) - return optional(); - - return e.block_id; - } - catch (const fc::exception&) - { - } - catch (const std::exception&) - { - } + optional entry = last_index_entry(); + if( entry.valid() ) return entry->block_id; return optional(); } - } } diff --git a/libraries/chain/include/graphene/chain/block_database.hpp b/libraries/chain/include/graphene/chain/block_database.hpp index d1f613c1..2c7ff812 100644 --- a/libraries/chain/include/graphene/chain/block_database.hpp +++ b/libraries/chain/include/graphene/chain/block_database.hpp @@ -26,6 +26,8 @@ #include namespace graphene { namespace chain { + class index_entry; + class block_database { public: @@ -44,6 +46,7 @@ namespace graphene { namespace chain { optional last()const; optional last_id()const; private: + optional last_index_entry()const; mutable std::fstream _blocks; mutable std::fstream _block_num_to_pos; }; From 17417037c648fd01529b3e894ff3377833cc3b1d Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Wed, 12 Jul 2017 22:03:57 +0200 Subject: [PATCH 03/16] Moved reindex logic into database / chain_database, make use of additional blocks in block_database Fixed tests wrt db.open --- libraries/app/application.cpp | 67 +++--------------- libraries/chain/db_management.cpp | 69 +++++++++++-------- .../chain/include/graphene/chain/database.hpp | 6 +- libraries/db/object_database.cpp | 7 ++ tests/benchmarks/genesis_allocation.cpp | 6 +- tests/common/database_fixture.cpp | 2 +- tests/generate_empty_blocks/main.cpp | 2 +- tests/tests/block_tests.cpp | 26 +++---- tests/tests/operation_tests2.cpp | 2 +- 9 files changed, 81 insertions(+), 106 deletions(-) diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index 5e4f9c7e..1a0b8b65 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -300,7 +300,6 @@ namespace detail { ~application_impl() { - fc::remove_all(_data_dir / "blockchain/dblock"); } void set_dbg_init_key( genesis_state_type& genesis, const std::string& init_key ) @@ -314,8 +313,7 @@ namespace detail { void startup() { try { - bool clean = !fc::exists(_data_dir / "blockchain/dblock"); - fc::create_directories(_data_dir / "blockchain/dblock"); + fc::create_directories(_data_dir / "blockchain"); auto initial_state = [&] { ilog("Initializing database..."); @@ -381,64 +379,17 @@ namespace detail { bool replay = false; std::string replay_reason = "reason not provided"; - // never replay if data dir is empty - if( fc::exists( _data_dir ) && fc::directory_iterator( _data_dir ) != fc::directory_iterator() ) - { - if( _options->count("replay-blockchain") ) - { - replay = true; - replay_reason = "replay-blockchain argument specified"; - } - else if( !clean ) - { - replay = true; - replay_reason = "unclean shutdown detected"; - } - else if( !fc::exists( _data_dir / "db_version" ) ) - { - replay = true; - replay_reason = "db_version file not found"; - } - else - { - std::string version_string; - fc::read_file_contents( _data_dir / "db_version", version_string ); + if( _options->count("replay-blockchain") ) + _chain_db->wipe( _data_dir / "blockchain", false ); - if( version_string != GRAPHENE_CURRENT_DB_VERSION ) - { - replay = true; - replay_reason = "db_version file content mismatch"; - } - } + try + { + _chain_db->open( _data_dir / "blockchain", initial_state, GRAPHENE_CURRENT_DB_VERSION ); } - - if( !replay ) + catch( const fc::exception& e ) { - try - { - _chain_db->open( _data_dir / "blockchain", initial_state ); - } - catch( const fc::exception& e ) - { - ilog( "Caught exception ${e} in open()", ("e", e.to_detail_string()) ); - - replay = true; - replay_reason = "exception in open()"; - } - } - - if( replay ) - { - ilog( "Replaying blockchain due to: ${reason}", ("reason", replay_reason) ); - - fc::remove_all( _data_dir / "db_version" ); - _chain_db->reindex( _data_dir / "blockchain", initial_state() ); - - const auto mode = std::ios::out | std::ios::binary | std::ios::trunc; - std::ofstream db_version( (_data_dir / "db_version").generic_string().c_str(), mode ); - std::string version_string = GRAPHENE_CURRENT_DB_VERSION; - db_version.write( version_string.c_str(), version_string.size() ); - db_version.close(); + elog( "Caught exception ${e} in open(), you might want to force a replay", ("e", e.to_detail_string()) ); + throw; } if( _options->count("force-validate") ) diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp index 68f6fad1..ca4004e6 100644 --- a/libraries/chain/db_management.cpp +++ b/libraries/chain/db_management.cpp @@ -47,33 +47,39 @@ database::~database() clear_pending(); } -void database::reindex(fc::path data_dir, const genesis_state_type& initial_allocation) +void database::reindex( fc::path data_dir ) { try { - ilog( "reindexing blockchain" ); - wipe(data_dir, false); - open(data_dir, [&initial_allocation]{return initial_allocation;}); - - auto start = fc::time_point::now(); auto last_block = _block_id_to_block.last(); if( !last_block ) { elog( "!no last block" ); edump((last_block)); return; } + if( last_block->block_num() <= head_block_num()) return; + ilog( "reindexing blockchain" ); + auto start = fc::time_point::now(); const auto last_block_num = last_block->block_num(); + uint32_t flush_point = last_block_num - 10000; + uint32_t undo_point = last_block_num - 50; ilog( "Replaying blocks..." ); - // Right now, we leave undo_db enabled when replaying when the bookie plugin is + // Right now, we leave undo_db enabled when replaying when the bookie plugin is // enabled. It depends on new/changed/removed object notifications, and those are // only fired when the undo_db is enabled if (!_slow_replays) _undo_db.disable(); - for( uint32_t i = 1; i <= last_block_num; ++i ) + for( uint32_t i = head_block_num() + 1; i <= last_block_num; ++i ) { - if( i == 1 || - i % 10000 == 0 ) - std::cerr << " " << double(i*100)/last_block_num << "% "<< i << " of " < block = _block_id_to_block.fetch_by_number(i); if( !block.valid() ) { @@ -127,10 +133,29 @@ void database::wipe(const fc::path& data_dir, bool include_blocks) void database::open( const fc::path& data_dir, - std::function genesis_loader) + std::function genesis_loader, + const std::string& db_version) { try { + bool wipe_object_db = false; + if( !fc::exists( data_dir / "db_version" ) ) + wipe_object_db = true; + else + { + std::string version_string; + fc::read_file_contents( data_dir / "db_version", version_string ); + wipe_object_db = ( version_string != db_version ); + } + if( wipe_object_db ) { + ilog("Wiping object_database due to missing or wrong version"); + object_database::wipe( data_dir ); + std::ofstream version_file( (data_dir / "db_version").generic_string().c_str(), + std::ios::out | std::ios::binary | std::ios::trunc ); + version_file.write( db_version.c_str(), db_version.size() ); + version_file.close(); + } + object_database::open(data_dir); _block_id_to_block.open(data_dir / "database" / "block_num_to_block"); @@ -138,15 +163,13 @@ void database::open( if( !find(global_property_id_type()) ) init_genesis(genesis_loader()); - fc::optional last_block = _block_id_to_block.last(); + fc::optional last_block = _block_id_to_block.last_id(); if( last_block.valid() ) { - _fork_db.start_block( *last_block ); - if( last_block->id() != head_block_id() ) - { - FC_ASSERT( head_block_num() == 0, "last block ID does not match current chain state", - ("last_block->id", last_block->id())("head_block_num",head_block_num()) ); - } + FC_ASSERT( *last_block >= head_block_id(), + "last block ID does not match current chain state", + ("last_block->id", last_block)("head_block_id",head_block_num()) ); + reindex( data_dir ); } } FC_CAPTURE_LOG_AND_RETHROW( (data_dir) ) @@ -167,17 +190,9 @@ void database::close(bool rewind) while( head_block_num() > cutoff ) { - // elog("pop"); block_id_type popped_block_id = head_block_id(); pop_block(); _fork_db.remove(popped_block_id); // doesn't throw on missing - try - { - _block_id_to_block.remove(popped_block_id); - } - catch (const fc::key_not_found_exception&) - { - } } } catch ( const fc::exception& e ) diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp index 0c6dcb0f..77699ee4 100644 --- a/libraries/chain/include/graphene/chain/database.hpp +++ b/libraries/chain/include/graphene/chain/database.hpp @@ -91,10 +91,12 @@ namespace graphene { namespace chain { * * @param data_dir Path to open or create database in * @param genesis_loader A callable object which returns the genesis state to initialize new databases on + * @param db_version a version string that changes when the internal database format and/or logic is modified */ void open( const fc::path& data_dir, - std::function genesis_loader ); + std::function genesis_loader, + const std::string& db_version ); /** * @brief Rebuild object graph from block history and open detabase @@ -102,7 +104,7 @@ namespace graphene { namespace chain { * This method may be called after or instead of @ref database::open, and will rebuild the object graph by * replaying blockchain history. When this method exits successfully, the database will be open. */ - void reindex(fc::path data_dir, const genesis_state_type& initial_allocation = genesis_state_type()); + void reindex(fc::path data_dir); /** * @brief wipe Delete database from disk, and potentially the raw chain as well. diff --git a/libraries/db/object_database.cpp b/libraries/db/object_database.cpp index 29d83ae7..6e1fdea2 100644 --- a/libraries/db/object_database.cpp +++ b/libraries/db/object_database.cpp @@ -71,6 +71,7 @@ index& object_database::get_mutable_index(uint8_t space_id, uint8_t type_id) void object_database::flush() { // ilog("Save object_database in ${d}", ("d", _data_dir)); + fc::create_directories( _data_dir / "object_database" / "lock" ); for( uint32_t space = 0; space < _index.size(); ++space ) { fc::create_directories( _data_dir / "object_database" / fc::to_string(space) ); @@ -79,6 +80,7 @@ void object_database::flush() if( _index[space][type] ) _index[space][type]->save( _data_dir / "object_database" / fc::to_string(space)/fc::to_string(type) ); } + fc::remove_all( _data_dir / "object_database" / "lock" ); } void object_database::wipe(const fc::path& data_dir) @@ -91,6 +93,11 @@ void object_database::wipe(const fc::path& data_dir) void object_database::open(const fc::path& data_dir) { try { + if( fc::exists( _data_dir / "object_database" / "lock" ) ) + { + wlog("Ignoring locked object_database"); + return; + } ilog("Opening object database from ${d} ...", ("d", data_dir)); _data_dir = data_dir; for( uint32_t space = 0; space < _index.size(); ++space ) diff --git a/tests/benchmarks/genesis_allocation.cpp b/tests/benchmarks/genesis_allocation.cpp index 61a3b1b8..a17a16fa 100644 --- a/tests/benchmarks/genesis_allocation.cpp +++ b/tests/benchmarks/genesis_allocation.cpp @@ -68,7 +68,7 @@ BOOST_AUTO_TEST_CASE( genesis_and_persistence_bench ) { database db; - db.open(data_dir.path(), [&]{return genesis_state;}); + db.open(data_dir.path(), [&]{return genesis_state;}, "test"); for( int i = 11; i < account_count + 11; ++i) BOOST_CHECK(db.get_balance(account_id_type(i), asset_id_type()).amount == GRAPHENE_MAX_SHARE_SUPPLY / account_count); @@ -81,7 +81,7 @@ BOOST_AUTO_TEST_CASE( genesis_and_persistence_bench ) database db; fc::time_point start_time = fc::time_point::now(); - db.open(data_dir.path(), [&]{return genesis_state;}); + db.open(data_dir.path(), [&]{return genesis_state;}, "test"); ilog("Opened database in ${t} milliseconds.", ("t", (fc::time_point::now() - start_time).count() / 1000)); for( int i = 11; i < account_count + 11; ++i) @@ -116,7 +116,7 @@ BOOST_AUTO_TEST_CASE( genesis_and_persistence_bench ) auto start_time = fc::time_point::now(); wlog( "about to start reindex..." ); - db.reindex(data_dir.path(), genesis_state); + db.open(data_dir.path(), [&]{return genesis_state;}, "force_wipe"); ilog("Replayed database in ${t} milliseconds.", ("t", (fc::time_point::now() - start_time).count() / 1000)); for( int i = 0; i < blocks_to_produce; ++i ) diff --git a/tests/common/database_fixture.cpp b/tests/common/database_fixture.cpp index e6a0b327..1741c787 100644 --- a/tests/common/database_fixture.cpp +++ b/tests/common/database_fixture.cpp @@ -355,7 +355,7 @@ void database_fixture::open_database() { if( !data_dir ) { data_dir = fc::temp_directory( graphene::utilities::temp_directory_path() ); - db.open(data_dir->path(), [this]{return genesis_state;}); + db.open(data_dir->path(), [this]{return genesis_state;}, "test"); } } diff --git a/tests/generate_empty_blocks/main.cpp b/tests/generate_empty_blocks/main.cpp index 1b45340d..b6a2ca95 100644 --- a/tests/generate_empty_blocks/main.cpp +++ b/tests/generate_empty_blocks/main.cpp @@ -124,7 +124,7 @@ int main( int argc, char** argv ) database db; fc::path db_path = data_dir / "db"; - db.open(db_path, [&]() { return genesis; } ); + db.open(db_path, [&]() { return genesis; }, "TEST" ); uint32_t slot = 1; uint32_t missed = 0; diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp index 07609d4b..5990b12b 100644 --- a/tests/tests/block_tests.cpp +++ b/tests/tests/block_tests.cpp @@ -138,7 +138,7 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) signed_block cutoff_block; { database db; - db.open(data_dir.path(), make_genesis ); + db.open(data_dir.path(), make_genesis, "TEST" ); b = db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); // TODO: Change this test when we correct #406 @@ -162,7 +162,7 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) } { database db; - db.open(data_dir.path(), []{return genesis_state_type();}); + db.open(data_dir.path(), []{return genesis_state_type();}, "TEST"); BOOST_CHECK_EQUAL( db.head_block_num(), cutoff_block.block_num() ); b = cutoff_block; for( uint32_t i = 0; i < 200; ++i ) @@ -187,7 +187,7 @@ BOOST_AUTO_TEST_CASE( undo_block ) fc::temp_directory data_dir( graphene::utilities::temp_directory_path() ); { database db; - db.open(data_dir.path(), make_genesis); + db.open(data_dir.path(), make_genesis, "TEST"); fc::time_point_sec now( GRAPHENE_TESTING_GENESIS_TIMESTAMP ); std::vector< time_point_sec > time_stack; @@ -236,9 +236,9 @@ BOOST_AUTO_TEST_CASE( fork_blocks ) fc::temp_directory data_dir2( graphene::utilities::temp_directory_path() ); database db1; - db1.open(data_dir1.path(), make_genesis); + db1.open(data_dir1.path(), make_genesis, "TEST"); database db2; - db2.open(data_dir2.path(), make_genesis); + db2.open(data_dir2.path(), make_genesis, "TEST"); BOOST_CHECK( db1.get_chain_id() == db2.get_chain_id() ); auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); @@ -381,7 +381,7 @@ BOOST_AUTO_TEST_CASE( undo_pending ) fc::temp_directory data_dir( graphene::utilities::temp_directory_path() ); { database db; - db.open(data_dir.path(), make_genesis); + db.open(data_dir.path(), make_genesis, "TEST"); auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); public_key_type init_account_pub_key = init_account_priv_key.get_public_key(); @@ -446,8 +446,8 @@ BOOST_AUTO_TEST_CASE( switch_forks_undo_create ) dir2( graphene::utilities::temp_directory_path() ); database db1, db2; - db1.open(dir1.path(), make_genesis); - db2.open(dir2.path(), make_genesis); + db1.open(dir1.path(), make_genesis, "TEST"); + db2.open(dir2.path(), make_genesis, "TEST"); BOOST_CHECK( db1.get_chain_id() == db2.get_chain_id() ); auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); @@ -505,8 +505,8 @@ BOOST_AUTO_TEST_CASE( duplicate_transactions ) dir2( graphene::utilities::temp_directory_path() ); database db1, db2; - db1.open(dir1.path(), make_genesis); - db2.open(dir2.path(), make_genesis); + db1.open(dir1.path(), make_genesis, "TEST"); + db2.open(dir2.path(), make_genesis, "TEST"); BOOST_CHECK( db1.get_chain_id() == db2.get_chain_id() ); auto skip_sigs = database::skip_transaction_signatures | database::skip_authority_check; @@ -555,7 +555,7 @@ BOOST_AUTO_TEST_CASE( tapos ) try { fc::temp_directory dir1( graphene::utilities::temp_directory_path() ); database db1; - db1.open(dir1.path(), make_genesis); + db1.open(dir1.path(), make_genesis, "TEST"); const account_object& init1 = *db1.get_index_type().indices().get().find("init1"); @@ -1106,7 +1106,7 @@ BOOST_FIXTURE_TEST_CASE( transaction_invalidated_in_cache, database_fixture ) fc::temp_directory data_dir2( graphene::utilities::temp_directory_path() ); database db2; - db2.open(data_dir2.path(), make_genesis); + db2.open(data_dir2.path(), make_genesis, "TEST"); BOOST_CHECK( db.get_chain_id() == db2.get_chain_id() ); while( db2.head_block_num() < db.head_block_num() ) @@ -1269,7 +1269,7 @@ BOOST_AUTO_TEST_CASE( genesis_reserve_ids ) genesis_state.initial_assets.push_back( usd ); return genesis_state; - } ); + }, "TEST" ); const auto& acct_idx = db.get_index_type().indices().get(); auto acct_itr = acct_idx.find("init0"); diff --git a/tests/tests/operation_tests2.cpp b/tests/tests/operation_tests2.cpp index 98d40207..3981270d 100644 --- a/tests/tests/operation_tests2.cpp +++ b/tests/tests/operation_tests2.cpp @@ -1111,7 +1111,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test ) auto _sign = [&]( signed_transaction& tx, const private_key_type& key ) { tx.sign( key, db.get_chain_id() ); }; - db.open(td.path(), [this]{return genesis_state;}); + db.open(td.path(), [this]{return genesis_state;}, "TEST"); const balance_object& balance = balance_id_type()(db); BOOST_CHECK_EQUAL(balance.balance.amount.value, 1); BOOST_CHECK_EQUAL(balance_id_type(1)(db).balance.amount.value, 1); From a0052d4bd35f2a0882a051a44b5b9f656317b9e3 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Thu, 13 Jul 2017 20:26:35 +0200 Subject: [PATCH 04/16] Enable undo + fork database for final blocks in a replay Dont remove blocks from block db when popping blocks, handle edge case in replay wrt fork_db, adapted unit tests --- libraries/chain/db_block.cpp | 1 - libraries/chain/db_management.cpp | 40 ++++++++++++++++++------------- tests/tests/block_tests.cpp | 6 ++++- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp index 9b2c7f36..ad4ad1a0 100644 --- a/libraries/chain/db_block.cpp +++ b/libraries/chain/db_block.cpp @@ -506,7 +506,6 @@ void database::pop_block() GRAPHENE_ASSERT( head_block.valid(), pop_empty_chain, "there are no blocks to pop" ); _fork_db.pop_block(); - _block_id_to_block.remove( head_id ); pop_undo(); _popped_tx.insert( _popped_tx.begin(), head_block->transactions.begin(), head_block->transactions.end() ); diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp index ca4004e6..1a60520a 100644 --- a/libraries/chain/db_management.cpp +++ b/libraries/chain/db_management.cpp @@ -64,11 +64,16 @@ void database::reindex( fc::path data_dir ) uint32_t undo_point = last_block_num - 50; ilog( "Replaying blocks..." ); - // Right now, we leave undo_db enabled when replaying when the bookie plugin is - // enabled. It depends on new/changed/removed object notifications, and those are - // only fired when the undo_db is enabled - if (!_slow_replays) - _undo_db.disable(); + if( head_block_num() >= undo_point ) + _fork_db.start_block( *fetch_block_by_number( head_block_num() ) ); + else + { + // Right now, we leave undo_db enabled when replaying when the bookie plugin is + // enabled. It depends on new/changed/removed object notifications, and those are + // only fired when the undo_db is enabled + if (!_slow_replays) + _undo_db.disable(); + } for( uint32_t i = head_block_num() + 1; i <= last_block_num; ++i ) { if( i % 10000 == 0 ) std::cerr << " " << double(i*100)/last_block_num << "% "< block = _block_id_to_block.fetch_by_number(i); if( !block.valid() ) { @@ -100,21 +103,26 @@ void database::reindex( fc::path data_dir ) wlog( "Dropped ${n} blocks from after the gap", ("n", dropped_count) ); break; } - if (_slow_replays) - push_block(*block, skip_fork_db | - skip_witness_signature | - skip_transaction_signatures | - skip_transaction_dupe_check | - skip_tapos_check | - skip_witness_schedule_check | - skip_authority_check); - else + if( i < undo_point && !_slow_replays) + { apply_block(*block, skip_witness_signature | skip_transaction_signatures | skip_transaction_dupe_check | skip_tapos_check | skip_witness_schedule_check | skip_authority_check); + } + else + { + if (!_slow_replays) + _undo_db.enable(); + push_block(*block, skip_witness_signature | + skip_transaction_signatures | + skip_transaction_dupe_check | + skip_tapos_check | + skip_witness_schedule_check | + skip_authority_check); + } } if (!_slow_replays) _undo_db.enable(); diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp index 5990b12b..daa0734b 100644 --- a/tests/tests/block_tests.cpp +++ b/tests/tests/block_tests.cpp @@ -136,6 +136,7 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) // TODO: Don't generate this here auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); signed_block cutoff_block; + uint32_t last_block; { database db; db.open(data_dir.path(), make_genesis, "TEST" ); @@ -155,6 +156,7 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) if( cutoff_height >= 200 ) { cutoff_block = *(db.fetch_block_by_number( cutoff_height )); + last_block = db.head_block_num(); break; } } @@ -163,7 +165,9 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) { database db; db.open(data_dir.path(), []{return genesis_state_type();}, "TEST"); - BOOST_CHECK_EQUAL( db.head_block_num(), cutoff_block.block_num() ); + BOOST_CHECK_EQUAL( db.head_block_num(), last_block ); + while( db.head_block_num() > cutoff_block.block_num() ) + db.pop_block(); b = cutoff_block; for( uint32_t i = 0; i < 200; ++i ) { From 3bee3f29a23aef02fcee520d5ab5cac6f2c490fc Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Mon, 31 Jul 2017 14:20:30 +0200 Subject: [PATCH 05/16] Log starting block number of replay --- libraries/chain/db_management.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp index 1a60520a..0f678910 100644 --- a/libraries/chain/db_management.cpp +++ b/libraries/chain/db_management.cpp @@ -63,7 +63,7 @@ void database::reindex( fc::path data_dir ) uint32_t flush_point = last_block_num - 10000; uint32_t undo_point = last_block_num - 50; - ilog( "Replaying blocks..." ); + ilog( "Replaying blocks, starting at ${next}...", ("next",head_block_num() + 1) ); if( head_block_num() >= undo_point ) _fork_db.start_block( *fetch_block_by_number( head_block_num() ) ); else From b45a6ca14732e73f69ba8feedc69280419be8d1e Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Mon, 31 Jul 2017 20:24:04 +0200 Subject: [PATCH 06/16] Prevent unsigned integer underflow --- libraries/chain/db_management.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp index 0f678910..c2ffbf0a 100644 --- a/libraries/chain/db_management.cpp +++ b/libraries/chain/db_management.cpp @@ -60,12 +60,15 @@ void database::reindex( fc::path data_dir ) ilog( "reindexing blockchain" ); auto start = fc::time_point::now(); const auto last_block_num = last_block->block_num(); - uint32_t flush_point = last_block_num - 10000; - uint32_t undo_point = last_block_num - 50; + uint32_t flush_point = last_block_num < 10000 ? 0 : last_block_num - 10000; + uint32_t undo_point = last_block_num < 50 ? 0 : last_block_num - 50; ilog( "Replaying blocks, starting at ${next}...", ("next",head_block_num() + 1) ); if( head_block_num() >= undo_point ) - _fork_db.start_block( *fetch_block_by_number( head_block_num() ) ); + { + if( head_block_num() > 0 ) + _fork_db.start_block( *fetch_block_by_number( head_block_num() ) ); + } else { // Right now, we leave undo_db enabled when replaying when the bookie plugin is From c8f8f1a44b21f1c1a7835be49c88783fc42e1c86 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Thu, 3 Aug 2017 15:41:40 +0200 Subject: [PATCH 07/16] Fixed lock detection --- libraries/db/object_database.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/db/object_database.cpp b/libraries/db/object_database.cpp index 6e1fdea2..9d251645 100644 --- a/libraries/db/object_database.cpp +++ b/libraries/db/object_database.cpp @@ -93,7 +93,7 @@ void object_database::wipe(const fc::path& data_dir) void object_database::open(const fc::path& data_dir) { try { - if( fc::exists( _data_dir / "object_database" / "lock" ) ) + if( fc::exists( data_dir / "object_database" / "lock" ) ) { wlog("Ignoring locked object_database"); return; From b71f20e06018f8c0d70fc3a559f41581b8ab3d7e Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Fri, 4 Aug 2017 17:55:13 +0200 Subject: [PATCH 08/16] Dont leave _data_dir empty if db is locked --- libraries/db/object_database.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/db/object_database.cpp b/libraries/db/object_database.cpp index 9d251645..03ded276 100644 --- a/libraries/db/object_database.cpp +++ b/libraries/db/object_database.cpp @@ -93,13 +93,13 @@ void object_database::wipe(const fc::path& data_dir) void object_database::open(const fc::path& data_dir) { try { - if( fc::exists( data_dir / "object_database" / "lock" ) ) + _data_dir = data_dir; + if( fc::exists( _data_dir / "object_database" / "lock" ) ) { wlog("Ignoring locked object_database"); return; } ilog("Opening object database from ${d} ...", ("d", data_dir)); - _data_dir = data_dir; for( uint32_t space = 0; space < _index.size(); ++space ) for( uint32_t type = 0; type < _index[space].size(); ++type ) if( _index[space][type] ) From 95a5b57c4feae99eb0bb1fa000140e8fd9701e73 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Fri, 4 Aug 2017 20:50:55 +0200 Subject: [PATCH 09/16] Writing the object_database is now almost atomic --- libraries/db/object_database.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/libraries/db/object_database.cpp b/libraries/db/object_database.cpp index 03ded276..fdde0fed 100644 --- a/libraries/db/object_database.cpp +++ b/libraries/db/object_database.cpp @@ -71,16 +71,20 @@ index& object_database::get_mutable_index(uint8_t space_id, uint8_t type_id) void object_database::flush() { // ilog("Save object_database in ${d}", ("d", _data_dir)); - fc::create_directories( _data_dir / "object_database" / "lock" ); + fc::create_directories( _data_dir / "object_database.tmp" / "lock" ); for( uint32_t space = 0; space < _index.size(); ++space ) { - fc::create_directories( _data_dir / "object_database" / fc::to_string(space) ); + fc::create_directories( _data_dir / "object_database.tmp" / fc::to_string(space) ); const auto types = _index[space].size(); for( uint32_t type = 0; type < types; ++type ) if( _index[space][type] ) - _index[space][type]->save( _data_dir / "object_database" / fc::to_string(space)/fc::to_string(type) ); + _index[space][type]->save( _data_dir / "object_database.tmp" / fc::to_string(space)/fc::to_string(type) ); } - fc::remove_all( _data_dir / "object_database" / "lock" ); + fc::remove_all( _data_dir / "object_database.tmp" / "lock" ); + if( fc::exists( _data_dir / "object_database" ) ) + fc::rename( _data_dir / "object_database", _data_dir / "object_database.old" ); + fc::rename( _data_dir / "object_database.tmp", _data_dir / "object_database" ); + fc::remove_all( _data_dir / "object_database.old" ); } void object_database::wipe(const fc::path& data_dir) From 0d108fb8ef005af065941571f909b263dd4c0ee3 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Fri, 4 Aug 2017 21:44:56 +0200 Subject: [PATCH 10/16] Improved consistency check for block_log --- libraries/chain/block_database.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/libraries/chain/block_database.cpp b/libraries/chain/block_database.cpp index 9c351624..87aeed28 100644 --- a/libraries/chain/block_database.cpp +++ b/libraries/chain/block_database.cpp @@ -216,23 +216,28 @@ optional block_database::last_index_entry()const { if( pos < sizeof(index_entry) ) return optional(); - if( pos % sizeof(index_entry) != 0 ) - pos -= pos % sizeof(index_entry); + pos -= pos % sizeof(index_entry); - while( e.block_size == 0 && pos > 0 ) + _blocks.seekg( 0, _block_num_to_pos.end ); + const std::streampos blocks_size = _blocks.tellg(); + while( pos >= 0 ) { pos -= sizeof(index_entry); _block_num_to_pos.seekg( pos ); _block_num_to_pos.read( (char*)&e, sizeof(e) ); - - if( e.block_size > 0 ) + if( _block_num_to_pos.gcount() == sizeof(e) && e.block_size > 0 + && e.block_pos + e.block_size <= blocks_size ) try { vector data( e.block_size ); _blocks.seekg( e.block_pos ); _blocks.read( data.data(), e.block_size ); - auto result = fc::raw::unpack(data); - return e; + if( _blocks.gcount() == e.block_size ) + { + const signed_block block = fc::raw::unpack(data); + if( block.id() == e.block_id ) + return e; + } } catch (const fc::exception&) { From ab382189fe1e4784d5afc4ef6b107b95d029e5ba Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sat, 5 Aug 2017 01:39:42 +0200 Subject: [PATCH 11/16] Cut back block_log index file if inconsistent --- libraries/chain/block_database.cpp | 12 +++++++----- .../chain/include/graphene/chain/block_database.hpp | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/libraries/chain/block_database.cpp b/libraries/chain/block_database.cpp index 87aeed28..3dcdcba4 100644 --- a/libraries/chain/block_database.cpp +++ b/libraries/chain/block_database.cpp @@ -45,14 +45,15 @@ void block_database::open( const fc::path& dbdir ) _block_num_to_pos.exceptions(std::ios_base::failbit | std::ios_base::badbit); _blocks.exceptions(std::ios_base::failbit | std::ios_base::badbit); - if( !fc::exists( dbdir/"index" ) ) + _index_filename = dbdir / "index"; + if( !fc::exists( _index_filename ) ) { - _block_num_to_pos.open( (dbdir/"index").generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out | std::fstream::trunc); + _block_num_to_pos.open( _index_filename.generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out | std::fstream::trunc); _blocks.open( (dbdir/"blocks").generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out | std::fstream::trunc); } else { - _block_num_to_pos.open( (dbdir/"index").generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out ); + _block_num_to_pos.open( _index_filename.generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out ); _blocks.open( (dbdir/"blocks").generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out ); } } FC_CAPTURE_AND_RETHROW( (dbdir) ) } @@ -121,7 +122,7 @@ bool block_database::contains( const block_id_type& id )const index_entry e; auto index_pos = sizeof(e)*block_header::num_from_id(id); _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); - if ( _block_num_to_pos.tellg() <= index_pos ) + if ( _block_num_to_pos.tellg() < index_pos + sizeof(e) ) return false; _block_num_to_pos.seekg( index_pos ); _block_num_to_pos.read( (char*)&e, sizeof(e) ); @@ -220,7 +221,7 @@ optional block_database::last_index_entry()const { _blocks.seekg( 0, _block_num_to_pos.end ); const std::streampos blocks_size = _blocks.tellg(); - while( pos >= 0 ) + while( pos > 0 ) { pos -= sizeof(index_entry); _block_num_to_pos.seekg( pos ); @@ -245,6 +246,7 @@ optional block_database::last_index_entry()const { catch (const std::exception&) { } + fc::resize_file( _index_filename, pos ); } } catch (const fc::exception&) diff --git a/libraries/chain/include/graphene/chain/block_database.hpp b/libraries/chain/include/graphene/chain/block_database.hpp index 2c7ff812..d902cd1b 100644 --- a/libraries/chain/include/graphene/chain/block_database.hpp +++ b/libraries/chain/include/graphene/chain/block_database.hpp @@ -47,6 +47,7 @@ namespace graphene { namespace chain { optional last_id()const; private: optional last_index_entry()const; + fc::path _index_filename; mutable std::fstream _blocks; mutable std::fstream _block_num_to_pos; }; From 7d0d61ab43af019ebd8e6cf7032bfe2de93563da Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sun, 6 Aug 2017 14:20:04 +0200 Subject: [PATCH 12/16] Fixed undo_database --- libraries/db/undo_database.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/libraries/db/undo_database.cpp b/libraries/db/undo_database.cpp index b37b2c7d..c5f2ef65 100644 --- a/libraries/db/undo_database.cpp +++ b/libraries/db/undo_database.cpp @@ -118,8 +118,6 @@ void undo_database::undo() _db.insert( std::move(*item.second) ); _stack.pop_back(); - if( _stack.empty() ) - _stack.emplace_back(); enable(); --_active_sessions; } FC_CAPTURE_AND_RETHROW() } @@ -127,6 +125,12 @@ void undo_database::undo() void undo_database::merge() { FC_ASSERT( _active_sessions > 0 ); + if( _active_sessions == 1 && _stack.size() == 1 ) + { + _stack.pop_back(); + --_active_sessions; + return; + } FC_ASSERT( _stack.size() >=2 ); auto& state = _stack.back(); auto& prev_state = _stack[_stack.size()-2]; From 731338f03c6d137164972264bfa279f201c1ce55 Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Sun, 6 Aug 2017 18:59:31 +0200 Subject: [PATCH 13/16] Added test case for broken merge on empty undo_db --- tests/tests/database_tests.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/tests/database_tests.cpp b/tests/tests/database_tests.cpp index 5dc35f27..0e2f1295 100644 --- a/tests/tests/database_tests.cpp +++ b/tests/tests/database_tests.cpp @@ -59,3 +59,22 @@ BOOST_AUTO_TEST_CASE( undo_test ) throw; } } + +BOOST_AUTO_TEST_CASE( merge_test ) +{ + try { + database db; + auto ses = db._undo_db.start_undo_session(); + const auto& bal_obj1 = db.create( [&]( account_balance_object& obj ){ + obj.balance = 42; + }); + ses.merge(); + + auto balance = db.get_balance( account_id_type(), asset_id_type() ); + BOOST_CHECK_EQUAL( 42, balance.amount.value ); + } catch ( const fc::exception& e ) + { + edump( (e.to_detail_string()) ); + throw; + } +} From 7b259ba2d3ea55bcdc4a8ab2629b78b6c0d1ec96 Mon Sep 17 00:00:00 2001 From: gladcow Date: Tue, 3 Sep 2019 08:07:46 +0300 Subject: [PATCH 14/16] exclude second undo_db.enable() call in some cases --- libraries/chain/db_management.cpp | 57 ++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp index c2ffbf0a..61f23db3 100644 --- a/libraries/chain/db_management.cpp +++ b/libraries/chain/db_management.cpp @@ -47,6 +47,50 @@ database::~database() clear_pending(); } +// Right now, we leave undo_db enabled when replaying when the bookie plugin is +// enabled. It depends on new/changed/removed object notifications, and those are +// only fired when the undo_db is enabled. +// So we use this helper object to disable undo_db only if it is not forbidden +// with _slow_replays flag. +class auto_undo_enabler +{ + const bool _slow_replays; + undo_database& _undo_db; + bool _disabled; +public: + auto_undo_enabler(bool slow_replays, undo_database& undo_db) : + _slow_replays(slow_replays), + _undo_db(undo_db), + _disabled(false) + { + } + + ~auto_undo_enabler() + { + try{ + enable(); + } FC_CAPTURE_AND_LOG(("undo_db enabling crash")) + } + + void enable() + { + if(!_disabled) + return; + _undo_db.enable(); + _disabled = false; + } + + void disable() + { + if(_disabled) + return; + if(_slow_replays) + return; + _undo_db.disable(); + _disabled = true; + } +}; + void database::reindex( fc::path data_dir ) { try { auto last_block = _block_id_to_block.last(); @@ -64,6 +108,7 @@ void database::reindex( fc::path data_dir ) uint32_t undo_point = last_block_num < 50 ? 0 : last_block_num - 50; ilog( "Replaying blocks, starting at ${next}...", ("next",head_block_num() + 1) ); + auto_undo_enabler undo(_slow_replays, _undo_db); if( head_block_num() >= undo_point ) { if( head_block_num() > 0 ) @@ -71,11 +116,7 @@ void database::reindex( fc::path data_dir ) } else { - // Right now, we leave undo_db enabled when replaying when the bookie plugin is - // enabled. It depends on new/changed/removed object notifications, and those are - // only fired when the undo_db is enabled - if (!_slow_replays) - _undo_db.disable(); + undo.disable(); } for( uint32_t i = head_block_num() + 1; i <= last_block_num; ++i ) { @@ -117,8 +158,7 @@ void database::reindex( fc::path data_dir ) } else { - if (!_slow_replays) - _undo_db.enable(); + undo.enable(); push_block(*block, skip_witness_signature | skip_transaction_signatures | skip_transaction_dupe_check | @@ -127,8 +167,7 @@ void database::reindex( fc::path data_dir ) skip_authority_check); } } - if (!_slow_replays) - _undo_db.enable(); + undo.enable(); auto end = fc::time_point::now(); ilog( "Done reindexing, elapsed time: ${t} sec", ("t",double((end-start).count())/1000000.0 ) ); } FC_CAPTURE_AND_RETHROW( (data_dir) ) } From f732048a6e253314eb521028f6260704a6ce0742 Mon Sep 17 00:00:00 2001 From: Abit Date: Thu, 24 May 2018 17:18:34 +0200 Subject: [PATCH 15/16] Merge pull request #938 from bitshares/fix-block-storing Store correct block ID when switching forks --- libraries/chain/db_block.cpp | 25 +++++--- tests/tests/block_tests.cpp | 119 +++++++++++++++++++++++++---------- 2 files changed, 104 insertions(+), 40 deletions(-) diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp index 9b2c7f36..97b00f86 100644 --- a/libraries/chain/db_block.cpp +++ b/libraries/chain/db_block.cpp @@ -215,12 +215,15 @@ bool database::_push_block(const signed_block& new_block) // pop blocks until we hit the forked block while( head_block_id() != branches.second.back()->data.previous ) + { + ilog( "popping block #${n} ${id}", ("n",head_block_num())("id",head_block_id()) ); pop_block(); + } // push all blocks on the new fork for( auto ritr = branches.first.rbegin(); ritr != branches.first.rend(); ++ritr ) { - ilog( "pushing blocks from fork ${n} ${id}", ("n",(*ritr)->data.block_num())("id",(*ritr)->data.id()) ); + ilog( "pushing block from fork #${n} ${id}", ("n",(*ritr)->data.block_num())("id",(*ritr)->id) ); optional except; try { undo_database::session session = _undo_db.start_undo_session(); @@ -235,21 +238,27 @@ bool database::_push_block(const signed_block& new_block) // remove the rest of branches.first from the fork_db, those blocks are invalid while( ritr != branches.first.rend() ) { - _fork_db.remove( (*ritr)->data.id() ); + ilog( "removing block from fork_db #${n} ${id}", ("n",(*ritr)->data.block_num())("id",(*ritr)->id) ); + _fork_db.remove( (*ritr)->id ); ++ritr; } _fork_db.set_head( branches.second.front() ); // pop all blocks from the bad fork while( head_block_id() != branches.second.back()->data.previous ) - pop_block(); - - // restore all blocks from the good fork - for( auto ritr = branches.second.rbegin(); ritr != branches.second.rend(); ++ritr ) { + ilog( "popping block #${n} ${id}", ("n",head_block_num())("id",head_block_id()) ); + pop_block(); + } + + ilog( "Switching back to fork: ${id}", ("id",branches.second.front()->data.id()) ); + // restore all blocks from the good fork + for( auto ritr2 = branches.second.rbegin(); ritr2 != branches.second.rend(); ++ritr2 ) + { + ilog( "pushing block #${n} ${id}", ("n",(*ritr2)->data.block_num())("id",(*ritr2)->id) ); auto session = _undo_db.start_undo_session(); - apply_block( (*ritr)->data, skip ); - _block_id_to_block.store( new_block.id(), (*ritr)->data ); + apply_block( (*ritr2)->data, skip ); + _block_id_to_block.store( (*ritr2)->id, (*ritr2)->data ); session.commit(); } throw *except; diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp index 07609d4b..7fce156a 100644 --- a/tests/tests/block_tests.cpp +++ b/tests/tests/block_tests.cpp @@ -242,51 +242,106 @@ BOOST_AUTO_TEST_CASE( fork_blocks ) BOOST_CHECK( db1.get_chain_id() == db2.get_chain_id() ); auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); - for( uint32_t i = 0; i < 10; ++i ) + + BOOST_TEST_MESSAGE( "Adding blocks 1 through 10" ); + for( uint32_t i = 1; i <= 10; ++i ) { auto b = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); try { PUSH_BLOCK( db2, b ); } FC_CAPTURE_AND_RETHROW( ("db2") ); } - for( uint32_t i = 10; i < 13; ++i ) + + for( uint32_t j = 0; j <= 4; j += 4 ) { - auto b = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); - } - string db1_tip = db1.head_block_id().str(); - uint32_t next_slot = 3; - for( uint32_t i = 13; i < 16; ++i ) - { - auto b = db2.generate_block(db2.get_slot_time(next_slot), db2.get_scheduled_witness(next_slot), init_account_priv_key, database::skip_nothing); - next_slot = 1; - // notify both databases of the new block. - // only db2 should switch to the new fork, db1 should not - PUSH_BLOCK( db1, b ); + // add blocks 11 through 13 to db1 only + BOOST_TEST_MESSAGE( "Adding 3 blocks to db1 only" ); + for( uint32_t i = 11 + j; i <= 13 + j; ++i ) + { + BOOST_TEST_MESSAGE( i ); + auto b = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + } + string db1_tip = db1.head_block_id().str(); + + // add different blocks 11 through 13 to db2 only + BOOST_TEST_MESSAGE( "Add 3 different blocks to db2 only" ); + uint32_t next_slot = 3; + for( uint32_t i = 11 + j; i <= 13 + j; ++i ) + { + BOOST_TEST_MESSAGE( i ); + auto b = db2.generate_block(db2.get_slot_time(next_slot), db2.get_scheduled_witness(next_slot), init_account_priv_key, database::skip_nothing); + next_slot = 1; + // notify both databases of the new block. + // only db2 should switch to the new fork, db1 should not + PUSH_BLOCK( db1, b ); + BOOST_CHECK_EQUAL(db1.head_block_id().str(), db1_tip); + BOOST_CHECK_EQUAL(db2.head_block_id().str(), b.id().str()); + } + + //The two databases are on distinct forks now, but at the same height. + BOOST_CHECK_EQUAL(db1.head_block_num(), 13u + j); + BOOST_CHECK_EQUAL(db2.head_block_num(), 13u + j); + BOOST_CHECK( db1.head_block_id() != db2.head_block_id() ); + + //Make a block on db2, make it invalid, then + //pass it to db1 and assert that db1 doesn't switch to the new fork. + signed_block good_block; + { + auto b = db2.generate_block(db2.get_slot_time(1), db2.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + good_block = b; + b.transactions.emplace_back(signed_transaction()); + b.transactions.back().operations.emplace_back(transfer_operation()); + b.sign( init_account_priv_key ); + BOOST_CHECK_EQUAL(b.block_num(), 14u + j); + GRAPHENE_CHECK_THROW(PUSH_BLOCK( db1, b ), fc::exception); + + // At this point, `fetch_block_by_number` will fetch block from fork_db, + // so unable to reproduce the issue which is fixed in PR #938 + // https://github.com/bitshares/bitshares-core/pull/938 + fc::optional previous_block = db1.fetch_block_by_number(1); + BOOST_CHECK ( previous_block.valid() ); + uint32_t db1_blocks = db1.head_block_num(); + for( uint32_t curr_block_num = 2; curr_block_num <= db1_blocks; ++curr_block_num ) + { + fc::optional curr_block = db1.fetch_block_by_number( curr_block_num ); + BOOST_CHECK( curr_block.valid() ); + BOOST_CHECK_EQUAL( curr_block->previous.str(), previous_block->id().str() ); + previous_block = curr_block; + } + } + BOOST_CHECK_EQUAL(db1.head_block_num(), 13u + j); BOOST_CHECK_EQUAL(db1.head_block_id().str(), db1_tip); - BOOST_CHECK_EQUAL(db2.head_block_id().str(), b.id().str()); + + if( j == 0 ) + { + // assert that db1 switches to new fork with good block + BOOST_CHECK_EQUAL(db2.head_block_num(), 14u + j); + PUSH_BLOCK( db1, good_block ); + BOOST_CHECK_EQUAL(db1.head_block_id().str(), db2.head_block_id().str()); + } } - //The two databases are on distinct forks now, but at the same height. Make a block on db2, make it invalid, then - //pass it to db1 and assert that db1 doesn't switch to the new fork. - signed_block good_block; - BOOST_CHECK_EQUAL(db1.head_block_num(), 13); - BOOST_CHECK_EQUAL(db2.head_block_num(), 13); + // generate more blocks to push the forked blocks out of fork_db + BOOST_TEST_MESSAGE( "Adding more blocks to db1, push the forked blocks out of fork_db" ); + for( uint32_t i = 1; i <= 50; ++i ) { - auto b = db2.generate_block(db2.get_slot_time(1), db2.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); - good_block = b; - b.transactions.emplace_back(signed_transaction()); - b.transactions.back().operations.emplace_back(transfer_operation()); - b.sign( init_account_priv_key ); - BOOST_CHECK_EQUAL(b.block_num(), 14); - GRAPHENE_CHECK_THROW(PUSH_BLOCK( db1, b ), fc::exception); + db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); } - BOOST_CHECK_EQUAL(db1.head_block_num(), 13); - BOOST_CHECK_EQUAL(db1.head_block_id().str(), db1_tip); - // assert that db1 switches to new fork with good block - BOOST_CHECK_EQUAL(db2.head_block_num(), 14); - PUSH_BLOCK( db1, good_block ); - BOOST_CHECK_EQUAL(db1.head_block_id().str(), db2.head_block_id().str()); + { + // PR #938 make sure db is in a good state https://github.com/bitshares/bitshares-core/pull/938 + BOOST_TEST_MESSAGE( "Checking whether all blocks on disk are good" ); + fc::optional previous_block = db1.fetch_block_by_number(1); + BOOST_CHECK ( previous_block.valid() ); + uint32_t db1_blocks = db1.head_block_num(); + for( uint32_t curr_block_num = 2; curr_block_num <= db1_blocks; ++curr_block_num ) + { + fc::optional curr_block = db1.fetch_block_by_number( curr_block_num ); + BOOST_CHECK( curr_block.valid() ); + BOOST_CHECK_EQUAL( curr_block->previous.str(), previous_block->id().str() ); + previous_block = curr_block; + } + } } catch (fc::exception& e) { edump((e.to_detail_string())); throw; From 22eb42e3ebf70d58cf73983e2c3c24ababf10820 Mon Sep 17 00:00:00 2001 From: Ronak Patel Date: Wed, 4 Sep 2019 15:14:18 +0530 Subject: [PATCH 16/16] Fixed integer overflow issue --- .../chain/include/graphene/chain/asset_object.hpp | 10 +++++++++- tests/tests/basic_tests.cpp | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/libraries/chain/include/graphene/chain/asset_object.hpp b/libraries/chain/include/graphene/chain/asset_object.hpp index afd5215a..f1df4681 100644 --- a/libraries/chain/include/graphene/chain/asset_object.hpp +++ b/libraries/chain/include/graphene/chain/asset_object.hpp @@ -230,8 +230,16 @@ namespace graphene { namespace chain { share_type settlement_fund; ///@} + /// The time when @ref current_feed would expire time_point_sec feed_expiration_time()const - { return current_feed_publication_time + options.feed_lifetime_sec; } + { + uint32_t current_feed_seconds = current_feed_publication_time.sec_since_epoch(); + if( std::numeric_limits::max() - current_feed_seconds <= options.feed_lifetime_sec ) + return time_point_sec::maximum(); + else + return current_feed_publication_time + options.feed_lifetime_sec; + } + bool feed_is_expired_before_hardfork_615(time_point_sec current_time)const { return feed_expiration_time() >= current_time; } bool feed_is_expired(time_point_sec current_time)const diff --git a/tests/tests/basic_tests.cpp b/tests/tests/basic_tests.cpp index 834c174b..da608541 100644 --- a/tests/tests/basic_tests.cpp +++ b/tests/tests/basic_tests.cpp @@ -540,4 +540,19 @@ BOOST_AUTO_TEST_CASE( merkle_root ) BOOST_CHECK( block.calculate_merkle_root() == c(dO) ); } +/** + * Reproduces https://github.com/bitshares/bitshares-core/issues/888 and tests fix for it. + */ +BOOST_AUTO_TEST_CASE( bitasset_feed_expiration_test ) +{ + time_point_sec now = fc::time_point::now(); + + asset_bitasset_data_object o; + + o.current_feed_publication_time = now - fc::hours(1); + o.options.feed_lifetime_sec = std::numeric_limits::max() - 1; + + BOOST_CHECK( !o.feed_is_expired( now ) ); +} + BOOST_AUTO_TEST_SUITE_END()