Peerplays_Blockchain/peerplays_migrated
1
0
Fork 0
Code Issues 35 Pull requests 4 Projects Releases 20 Packages Wiki Activity Actions

Moved reindex logic into database / chain_database, make use of additional blocks in block_database

Browse source 
Fixed tests wrt db.open
This commit is contained in:
Peter Conrad 2017-07-12 22:03:57 +02:00 committed by gladcow
parent 42680456b6
commit 17417037c6
9 changed files with 81 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

67
libraries/app/application.cpp
View file

@ -300,7 +300,6 @@ namespace detail {
~application_impl()
{
fc::remove_all(_data_dir / "blockchain/dblock");
}
void set_dbg_init_key( genesis_state_type& genesis, const std::string& init_key )
@ -314,8 +313,7 @@ namespace detail {
void startup()
{ try {
bool clean = !fc::exists(_data_dir / "blockchain/dblock");
fc::create_directories(_data_dir / "blockchain/dblock");
fc::create_directories(_data_dir / "blockchain");
auto initial_state = [&] {
ilog("Initializing database...");
@ -381,64 +379,17 @@ namespace detail {
bool replay = false;
std::string replay_reason = "reason not provided";
// never replay if data dir is empty
if( fc::exists( _data_dir ) && fc::directory_iterator( _data_dir ) != fc::directory_iterator() )
{
if( _options->count("replay-blockchain") )
{
replay = true;
replay_reason = "replay-blockchain argument specified";
}
else if( !clean )
{
replay = true;
replay_reason = "unclean shutdown detected";
}
else if( !fc::exists( _data_dir / "db_version" ) )
{
replay = true;
replay_reason = "db_version file not found";
}
else
{
std::string version_string;
fc::read_file_contents( _data_dir / "db_version", version_string );
if( _options->count("replay-blockchain") )
_chain_db->wipe( _data_dir / "blockchain", false );
if( version_string != GRAPHENE_CURRENT_DB_VERSION )
{
replay = true;
replay_reason = "db_version file content mismatch";
}
}
try
{
_chain_db->open( _data_dir / "blockchain", initial_state, GRAPHENE_CURRENT_DB_VERSION );
}
if( !replay )
catch( const fc::exception& e )
{
try
{
_chain_db->open( _data_dir / "blockchain", initial_state );
}
catch( const fc::exception& e )
{
ilog( "Caught exception ${e} in open()", ("e", e.to_detail_string()) );
replay = true;
replay_reason = "exception in open()";
}
}
if( replay )
{
ilog( "Replaying blockchain due to: ${reason}", ("reason", replay_reason) );
fc::remove_all( _data_dir / "db_version" );
_chain_db->reindex( _data_dir / "blockchain", initial_state() );
const auto mode = std::ios::out | std::ios::binary | std::ios::trunc;
std::ofstream db_version( (_data_dir / "db_version").generic_string().c_str(), mode );
std::string version_string = GRAPHENE_CURRENT_DB_VERSION;
db_version.write( version_string.c_str(), version_string.size() );
db_version.close();
elog( "Caught exception ${e} in open(), you might want to force a replay", ("e", e.to_detail_string()) );
throw;
}
if( _options->count("force-validate") )

69
libraries/chain/db_management.cpp
View file

@ -47,33 +47,39 @@ database::~database()
clear_pending();
}
void database::reindex(fc::path data_dir, const genesis_state_type& initial_allocation)
void database::reindex( fc::path data_dir )
{ try {
ilog( "reindexing blockchain" );
wipe(data_dir, false);
open(data_dir, [&initial_allocation]{return initial_allocation;});
auto start = fc::time_point::now();
auto last_block = _block_id_to_block.last();
if( !last_block ) {
elog( "!no last block" );
edump((last_block));
return;
}
if( last_block->block_num() <= head_block_num()) return;
ilog( "reindexing blockchain" );
auto start = fc::time_point::now();
const auto last_block_num = last_block->block_num();
uint32_t flush_point = last_block_num - 10000;
uint32_t undo_point = last_block_num - 50;
ilog( "Replaying blocks..." );
// Right now, we leave undo_db enabled when replaying when the bookie plugin is
// Right now, we leave undo_db enabled when replaying when the bookie plugin is
// enabled. It depends on new/changed/removed object notifications, and those are
// only fired when the undo_db is enabled
if (!_slow_replays)
_undo_db.disable();
for( uint32_t i = 1; i <= last_block_num; ++i )
for( uint32_t i = head_block_num() + 1; i <= last_block_num; ++i )
{
if( i == 1 ||
i % 10000 == 0 )
std::cerr << " " << double(i*100)/last_block_num << "% "<< i << " of " <<last_block_num<<" \n";
if( i % 10000 == 0 ) std::cerr << " " << double(i*100)/last_block_num << "% "<<i << " of " <<last_block_num<<" \n";
if( i == flush_point )
{
ilog( "Writing database to disk at block ${i}", ("i",i) );
flush();
ilog( "Done" );
}
if( i == undo_point )
_undo_db.enable();
fc::optional< signed_block > block = _block_id_to_block.fetch_by_number(i);
if( !block.valid() )
{
@ -127,10 +133,29 @@ void database::wipe(const fc::path& data_dir, bool include_blocks)
void database::open(
const fc::path& data_dir,
std::function<genesis_state_type()> genesis_loader)
std::function<genesis_state_type()> genesis_loader,
const std::string& db_version)
{
try
{
bool wipe_object_db = false;
if( !fc::exists( data_dir / "db_version" ) )
wipe_object_db = true;
else
{
std::string version_string;
fc::read_file_contents( data_dir / "db_version", version_string );
wipe_object_db = ( version_string != db_version );
}
if( wipe_object_db ) {
ilog("Wiping object_database due to missing or wrong version");
object_database::wipe( data_dir );
std::ofstream version_file( (data_dir / "db_version").generic_string().c_str(),
std::ios::out | std::ios::binary | std::ios::trunc );
version_file.write( db_version.c_str(), db_version.size() );
version_file.close();
}
object_database::open(data_dir);
_block_id_to_block.open(data_dir / "database" / "block_num_to_block");
@ -138,15 +163,13 @@ void database::open(
if( !find(global_property_id_type()) )
init_genesis(genesis_loader());
fc::optional<signed_block> last_block = _block_id_to_block.last();
fc::optional<block_id_type> last_block = _block_id_to_block.last_id();
if( last_block.valid() )
{
_fork_db.start_block( *last_block );
if( last_block->id() != head_block_id() )
{
FC_ASSERT( head_block_num() == 0, "last block ID does not match current chain state",
("last_block->id", last_block->id())("head_block_num",head_block_num()) );
}
FC_ASSERT( *last_block >= head_block_id(),
"last block ID does not match current chain state",
("last_block->id", last_block)("head_block_id",head_block_num()) );
reindex( data_dir );
}
}
FC_CAPTURE_LOG_AND_RETHROW( (data_dir) )
@ -167,17 +190,9 @@ void database::close(bool rewind)
while( head_block_num() > cutoff )
{
// elog("pop");
block_id_type popped_block_id = head_block_id();
pop_block();
_fork_db.remove(popped_block_id); // doesn't throw on missing
try
{
_block_id_to_block.remove(popped_block_id);
}
catch (const fc::key_not_found_exception&)
{
}
}
}
catch ( const fc::exception& e )

6
libraries/chain/include/graphene/chain/database.hpp
View file

@ -91,10 +91,12 @@ namespace graphene { namespace chain {
*
* @param data_dir Path to open or create database in
* @param genesis_loader A callable object which returns the genesis state to initialize new databases on
* @param db_version a version string that changes when the internal database format and/or logic is modified
*/
void open(
const fc::path& data_dir,
std::function<genesis_state_type()> genesis_loader );
std::function<genesis_state_type()> genesis_loader,
const std::string& db_version );
/**
* @brief Rebuild object graph from block history and open detabase
@ -102,7 +104,7 @@ namespace graphene { namespace chain {
* This method may be called after or instead of @ref database::open, and will rebuild the object graph by
* replaying blockchain history. When this method exits successfully, the database will be open.
*/
void reindex(fc::path data_dir, const genesis_state_type& initial_allocation = genesis_state_type());
void reindex(fc::path data_dir);
/**
* @brief wipe Delete database from disk, and potentially the raw chain as well.

7
libraries/db/object_database.cpp
View file

@ -71,6 +71,7 @@ index& object_database::get_mutable_index(uint8_t space_id, uint8_t type_id)
void object_database::flush()
{
// ilog("Save object_database in ${d}", ("d", _data_dir));
fc::create_directories( _data_dir / "object_database" / "lock" );
for( uint32_t space = 0; space < _index.size(); ++space )
{
fc::create_directories( _data_dir / "object_database" / fc::to_string(space) );
@ -79,6 +80,7 @@ void object_database::flush()
if( _index[space][type] )
_index[space][type]->save( _data_dir / "object_database" / fc::to_string(space)/fc::to_string(type) );
}
fc::remove_all( _data_dir / "object_database" / "lock" );
}
void object_database::wipe(const fc::path& data_dir)
@ -91,6 +93,11 @@ void object_database::wipe(const fc::path& data_dir)
void object_database::open(const fc::path& data_dir)
{ try {
if( fc::exists( _data_dir / "object_database" / "lock" ) )
{
wlog("Ignoring locked object_database");
return;
}
ilog("Opening object database from ${d} ...", ("d", data_dir));
_data_dir = data_dir;
for( uint32_t space = 0; space < _index.size(); ++space )

6
tests/benchmarks/genesis_allocation.cpp
View file

@ -68,7 +68,7 @@ BOOST_AUTO_TEST_CASE( genesis_and_persistence_bench )
{
database db;
db.open(data_dir.path(), [&]{return genesis_state;});
db.open(data_dir.path(), [&]{return genesis_state;}, "test");
for( int i = 11; i < account_count + 11; ++i)
BOOST_CHECK(db.get_balance(account_id_type(i), asset_id_type()).amount == GRAPHENE_MAX_SHARE_SUPPLY / account_count);
@ -81,7 +81,7 @@ BOOST_AUTO_TEST_CASE( genesis_and_persistence_bench )
database db;
fc::time_point start_time = fc::time_point::now();
db.open(data_dir.path(), [&]{return genesis_state;});
db.open(data_dir.path(), [&]{return genesis_state;}, "test");
ilog("Opened database in ${t} milliseconds.", ("t", (fc::time_point::now() - start_time).count() / 1000));
for( int i = 11; i < account_count + 11; ++i)
@ -116,7 +116,7 @@ BOOST_AUTO_TEST_CASE( genesis_and_persistence_bench )
auto start_time = fc::time_point::now();
wlog( "about to start reindex..." );
db.reindex(data_dir.path(), genesis_state);
db.open(data_dir.path(), [&]{return genesis_state;}, "force_wipe");
ilog("Replayed database in ${t} milliseconds.", ("t", (fc::time_point::now() - start_time).count() / 1000));
for( int i = 0; i < blocks_to_produce; ++i )

2
tests/common/database_fixture.cpp
View file

@ -355,7 +355,7 @@ void database_fixture::open_database()
{
if( !data_dir ) {
data_dir = fc::temp_directory( graphene::utilities::temp_directory_path() );
db.open(data_dir->path(), [this]{return genesis_state;});
db.open(data_dir->path(), [this]{return genesis_state;}, "test");
}
}

2
tests/generate_empty_blocks/main.cpp
View file

@ -124,7 +124,7 @@ int main( int argc, char** argv )
database db;
fc::path db_path = data_dir / "db";
db.open(db_path, [&]() { return genesis; } );
db.open(db_path, [&]() { return genesis; }, "TEST" );
uint32_t slot = 1;
uint32_t missed = 0;

26
tests/tests/block_tests.cpp
View file

@ -138,7 +138,7 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks )
signed_block cutoff_block;
{
database db;
db.open(data_dir.path(), make_genesis );
db.open(data_dir.path(), make_genesis, "TEST" );
b = db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing);
// TODO: Change this test when we correct #406
@ -162,7 +162,7 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks )
}
{
database db;
db.open(data_dir.path(), []{return genesis_state_type();});
db.open(data_dir.path(), []{return genesis_state_type();}, "TEST");
BOOST_CHECK_EQUAL( db.head_block_num(), cutoff_block.block_num() );
b = cutoff_block;
for( uint32_t i = 0; i < 200; ++i )
@ -187,7 +187,7 @@ BOOST_AUTO_TEST_CASE( undo_block )
fc::temp_directory data_dir( graphene::utilities::temp_directory_path() );
{
database db;
db.open(data_dir.path(), make_genesis);
db.open(data_dir.path(), make_genesis, "TEST");
fc::time_point_sec now( GRAPHENE_TESTING_GENESIS_TIMESTAMP );
std::vector< time_point_sec > time_stack;
@ -236,9 +236,9 @@ BOOST_AUTO_TEST_CASE( fork_blocks )
fc::temp_directory data_dir2( graphene::utilities::temp_directory_path() );
database db1;
db1.open(data_dir1.path(), make_genesis);
db1.open(data_dir1.path(), make_genesis, "TEST");
database db2;
db2.open(data_dir2.path(), make_genesis);
db2.open(data_dir2.path(), make_genesis, "TEST");
BOOST_CHECK( db1.get_chain_id() == db2.get_chain_id() );
auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) );
@ -381,7 +381,7 @@ BOOST_AUTO_TEST_CASE( undo_pending )
fc::temp_directory data_dir( graphene::utilities::temp_directory_path() );
{
database db;
db.open(data_dir.path(), make_genesis);
db.open(data_dir.path(), make_genesis, "TEST");
auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) );
public_key_type init_account_pub_key = init_account_priv_key.get_public_key();
@ -446,8 +446,8 @@ BOOST_AUTO_TEST_CASE( switch_forks_undo_create )
dir2( graphene::utilities::temp_directory_path() );
database db1,
db2;
db1.open(dir1.path(), make_genesis);
db2.open(dir2.path(), make_genesis);
db1.open(dir1.path(), make_genesis, "TEST");
db2.open(dir2.path(), make_genesis, "TEST");
BOOST_CHECK( db1.get_chain_id() == db2.get_chain_id() );
auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) );
@ -505,8 +505,8 @@ BOOST_AUTO_TEST_CASE( duplicate_transactions )
dir2( graphene::utilities::temp_directory_path() );
database db1,
db2;
db1.open(dir1.path(), make_genesis);
db2.open(dir2.path(), make_genesis);
db1.open(dir1.path(), make_genesis, "TEST");
db2.open(dir2.path(), make_genesis, "TEST");
BOOST_CHECK( db1.get_chain_id() == db2.get_chain_id() );
auto skip_sigs = database::skip_transaction_signatures | database::skip_authority_check;
@ -555,7 +555,7 @@ BOOST_AUTO_TEST_CASE( tapos )
try {
fc::temp_directory dir1( graphene::utilities::temp_directory_path() );
database db1;
db1.open(dir1.path(), make_genesis);
db1.open(dir1.path(), make_genesis, "TEST");
const account_object& init1 = *db1.get_index_type<account_index>().indices().get<by_name>().find("init1");
@ -1106,7 +1106,7 @@ BOOST_FIXTURE_TEST_CASE( transaction_invalidated_in_cache, database_fixture )
fc::temp_directory data_dir2( graphene::utilities::temp_directory_path() );
database db2;
db2.open(data_dir2.path(), make_genesis);
db2.open(data_dir2.path(), make_genesis, "TEST");
BOOST_CHECK( db.get_chain_id() == db2.get_chain_id() );
while( db2.head_block_num() < db.head_block_num() )
@ -1269,7 +1269,7 @@ BOOST_AUTO_TEST_CASE( genesis_reserve_ids )
genesis_state.initial_assets.push_back( usd );
return genesis_state;
} );
}, "TEST" );
const auto& acct_idx = db.get_index_type<account_index>().indices().get<by_name>();
auto acct_itr = acct_idx.find("init0");

2
tests/tests/operation_tests2.cpp
View file

@ -1111,7 +1111,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test )
auto _sign = [&]( signed_transaction& tx, const private_key_type& key )
{ tx.sign( key, db.get_chain_id() ); };
db.open(td.path(), [this]{return genesis_state;});
db.open(td.path(), [this]{return genesis_state;}, "TEST");
const balance_object& balance = balance_id_type()(db);
BOOST_CHECK_EQUAL(balance.balance.amount.value, 1);
BOOST_CHECK_EQUAL(balance_id_type(1)(db).balance.amount.value, 1);