From 9080800c5b61dbd0f374fe7ce3495e0e8e9cf657 Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Mon, 7 Sep 2015 17:46:47 -0400 Subject: [PATCH 01/18] Updated APIs, fixed crash - update fc to fix crash in websocket message handling - added api to verify transactions without broadcasting them - added api to broadcast a block produced outside the P2P network or witness --- libraries/app/api.cpp | 13 +++++++++ libraries/app/application.cpp | 5 ++++ libraries/app/include/graphene/app/api.hpp | 9 ++++++ libraries/chain/db_block.cpp | 6 ++++ .../chain/include/graphene/chain/database.hpp | 7 +++++ libraries/fc | 2 +- programs/witness_node/main.cpp | 29 +++++++++++-------- 7 files changed, 58 insertions(+), 13 deletions(-) diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index 5fc139fe..306e678b 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -629,6 +629,12 @@ namespace graphene { namespace app { _app.p2p_node()->broadcast_transaction(trx); } + void network_broadcast_api::broadcast_block( const signed_block& b ) + { + _app.chain_database()->push_block(b); + _app.p2p_node()->broadcast( net::block_message( b )); + } + void network_broadcast_api::broadcast_transaction_with_callback(confirmation_callback cb, const signed_transaction& trx) { trx.validate(); @@ -1214,6 +1220,13 @@ namespace graphene { namespace app { wdump((result)); return result; } + /** + * Validates a transaction against the current state without broadcast it on the network. + */ + processed_transaction database_api::validate_transaction( const signed_transaction& trx )const + { + return _db.validate_transaction(trx); + } bool database_api::verify_authority( const signed_transaction& trx )const { diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index 70b2899a..c9783d53 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -708,6 +708,11 @@ void application::shutdown_plugins() entry.second->plugin_shutdown(); return; } +void application::shutdown() +{ + if( my->_chain_db ) + my->_chain_db->close(); +} void application::initialize_plugins( const boost::program_options::variables_map& options ) { diff --git a/libraries/app/include/graphene/app/api.hpp b/libraries/app/include/graphene/app/api.hpp index ed495010..4100a62f 100644 --- a/libraries/app/include/graphene/app/api.hpp +++ b/libraries/app/include/graphene/app/api.hpp @@ -347,6 +347,11 @@ namespace graphene { namespace app { */ bool verify_account_authority( const string& name_or_id, const flat_set& signers )const; + /** + * Validates a transaction against the current state without broadcast it on the network. + */ + processed_transaction validate_transaction( const signed_transaction& trx )const; + /** * @return the set of blinded balance objects by commitment ID @@ -462,6 +467,8 @@ namespace graphene { namespace app { */ void broadcast_transaction_with_callback( confirmation_callback cb, const signed_transaction& trx); + void broadcast_block( const signed_block& block ); + /** * @brief Not reflected, thus not accessible to API clients. * @@ -595,6 +602,7 @@ FC_API(graphene::app::database_api, (get_blinded_balances) (get_required_fees) (set_subscribe_callback) + (validate_transaction) ) FC_API(graphene::app::history_api, (get_account_history) @@ -604,6 +612,7 @@ FC_API(graphene::app::history_api, FC_API(graphene::app::network_broadcast_api, (broadcast_transaction) (broadcast_transaction_with_callback) + (broadcast_block) ) FC_API(graphene::app::network_node_api, (add_node) diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp index fa93b8ed..0181cfa0 100644 --- a/libraries/chain/db_block.cpp +++ b/libraries/chain/db_block.cpp @@ -213,6 +213,12 @@ processed_transaction database::_push_transaction( const signed_transaction& trx return processed_trx; } +processed_transaction database::validate_transaction( const signed_transaction& trx ) +{ + auto session = _undo_db.start_undo_session(); + return _apply_transaction( trx ); +} + processed_transaction database::push_proposal(const proposal_object& proposal) { transaction_evaluation_state eval_state(this); diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp index 43056a73..c3fc26c8 100644 --- a/libraries/chain/include/graphene/chain/database.hpp +++ b/libraries/chain/include/graphene/chain/database.hpp @@ -404,7 +404,13 @@ namespace graphene { namespace chain { asset calculate_market_fee(const asset_object& recv_asset, const asset& trade_amount); asset pay_market_fees( const asset_object& recv_asset, const asset& receives ); + ///@} + /** + * This method validates transactions without adding it to the pending state. + * @return true if the transaction would validate + */ + processed_transaction validate_transaction( const signed_transaction& trx ); /** * @} @@ -429,6 +435,7 @@ namespace graphene { namespace chain { processed_transaction _apply_transaction( const signed_transaction& trx ); operation_result apply_operation( transaction_evaluation_state& eval_state, const operation& op ); + ///Steps involved in applying a new block ///@{ diff --git a/libraries/fc b/libraries/fc index c89a25a5..19e42ac4 160000 --- a/libraries/fc +++ b/libraries/fc @@ -1 +1 @@ -Subproject commit c89a25a55c333bd202185cebf3f87eeae2b54761 +Subproject commit 19e42ac4c41d0a2bbdc8094c2efeed5e28e0ed75 diff --git a/programs/witness_node/main.cpp b/programs/witness_node/main.cpp index 74126d39..97c7a4a9 100644 --- a/programs/witness_node/main.cpp +++ b/programs/witness_node/main.cpp @@ -53,8 +53,8 @@ void write_default_logging_config_to_stream(std::ostream& out); fc::optional load_logging_config_from_ini_file(const fc::path& config_ini_filename); int main(int argc, char** argv) { + app::application* node = new app::application(); try { - app::application node; bpo::options_description app_options("Graphene Witness Node"); bpo::options_description cfg_options("Graphene Witness Node"); app_options.add_options() @@ -64,14 +64,14 @@ int main(int argc, char** argv) { bpo::variables_map options; - auto witness_plug = node.register_plugin(); - auto history_plug = node.register_plugin(); - auto market_history_plug = node.register_plugin(); + auto witness_plug = node->register_plugin(); + auto history_plug = node->register_plugin(); + auto market_history_plug = node->register_plugin(); try { bpo::options_description cli, cfg; - node.set_program_options(cli, cfg); + node->set_program_options(cli, cfg); app_options.add(cli); cfg_options.add(cfg); bpo::store(bpo::parse_command_line(argc, argv, app_options), options); @@ -151,26 +151,31 @@ int main(int argc, char** argv) { } bpo::notify(options); - node.initialize(data_dir, options); - node.initialize_plugins( options ); + node->initialize(data_dir, options); + node->initialize_plugins( options ); - node.startup(); - node.startup_plugins(); + node->startup(); + node->startup_plugins(); fc::promise::ptr exit_promise = new fc::promise("UNIX Signal Handler"); fc::set_signal_handler([&exit_promise](int signal) { + elog( "Caught ^C attempting to exit cleanly" ); exit_promise->set_value(signal); }, SIGINT); - ilog("Started witness node on a chain with ${h} blocks.", ("h", node.chain_database()->head_block_num())); - ilog("Chain ID is ${id}", ("id", node.chain_database()->get_chain_id()) ); + ilog("Started witness node on a chain with ${h} blocks.", ("h", node->chain_database()->head_block_num())); + ilog("Chain ID is ${id}", ("id", node->chain_database()->get_chain_id()) ); int signal = exit_promise->wait(); ilog("Exiting from signal ${n}", ("n", signal)); - node.shutdown_plugins(); + node->shutdown_plugins(); + node->shutdown(); + delete node; return 0; } catch( const fc::exception& e ) { elog("Exiting with error:\n${e}", ("e", e.to_detail_string())); + node->shutdown(); + delete node; return 1; } } From a748883fed08a73afaec647e2e8827d07ca3c163 Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Mon, 7 Sep 2015 18:05:43 -0400 Subject: [PATCH 02/18] adding API callback for observing pending transactions --- libraries/app/api.cpp | 4 ++++ libraries/app/include/graphene/app/api.hpp | 4 ++++ libraries/chain/db_block.cpp | 3 +++ libraries/chain/include/graphene/chain/database.hpp | 6 ++++++ 4 files changed, 17 insertions(+) diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index 306e678b..5b61bc61 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -45,6 +45,10 @@ namespace graphene { namespace app { on_objects_removed(objs); }); _applied_block_connection = _db.applied_block.connect([this](const signed_block&){ on_applied_block(); }); + + _pending_trx_connection = _db.on_pending_transaction.connect([this](const signed_transaction& trx ){ + if( _pending_trx_callback ) _pending_trx_callback( fc::variant(trx) ); + }); } database_api::~database_api() diff --git a/libraries/app/include/graphene/app/api.hpp b/libraries/app/include/graphene/app/api.hpp index 4100a62f..ca83accd 100644 --- a/libraries/app/include/graphene/app/api.hpp +++ b/libraries/app/include/graphene/app/api.hpp @@ -366,6 +366,7 @@ namespace graphene { namespace app { vector get_required_fees( const vector& ops, asset_id_type id = asset_id_type() )const; void set_subscribe_callback( std::function cb, bool clear_filter ); + void set_pending_transaction_callback( std::function cb ){ _pending_trx_callback = cb; } private: template void subscribe_to_item( const T& i )const @@ -396,10 +397,12 @@ namespace graphene { namespace app { mutable fc::bloom_filter _subscribe_filter; std::function _subscribe_callback; + std::function _pending_trx_callback; boost::signals2::scoped_connection _change_connection; boost::signals2::scoped_connection _removed_connection; boost::signals2::scoped_connection _applied_block_connection; + boost::signals2::scoped_connection _pending_trx_connection; map< pair, std::function > _market_subscriptions; graphene::chain::database& _db; }; @@ -602,6 +605,7 @@ FC_API(graphene::app::database_api, (get_blinded_balances) (get_required_fees) (set_subscribe_callback) + (set_pending_transaction_callback) (validate_transaction) ) FC_API(graphene::app::history_api, diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp index 0181cfa0..5985e0fa 100644 --- a/libraries/chain/db_block.cpp +++ b/libraries/chain/db_block.cpp @@ -210,6 +210,9 @@ processed_transaction database::_push_transaction( const signed_transaction& trx notify_changed_objects(); // The transaction applied successfully. Merge its changes into the pending block session. session.merge(); + + // notify anyone listening to pending transactions + on_pending_transaction( trx ); return processed_trx; } diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp index c3fc26c8..a601b49c 100644 --- a/libraries/chain/include/graphene/chain/database.hpp +++ b/libraries/chain/include/graphene/chain/database.hpp @@ -196,6 +196,12 @@ namespace graphene { namespace chain { */ fc::signal applied_block; + /** + * This signal is emitted any time a new transaction is added to the pending + * block state. + */ + fc::signal on_pending_transaction; + /** * Emitted After a block has been applied and committed. The callback * should not yield and should execute quickly. From 8ebc1cfc43e97060f19dae78fbaa3f361fd30aa8 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Mon, 7 Sep 2015 12:43:50 -0400 Subject: [PATCH 03/18] node.cpp: Don't dump empty iterators --- libraries/net/node.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/net/node.cpp b/libraries/net/node.cpp index 124e0e13..6a8ef753 100644 --- a/libraries/net/node.cpp +++ b/libraries/net/node.cpp @@ -1199,9 +1199,9 @@ namespace graphene { namespace net { namespace detail { wdump((inventory_to_advertise)); for (const item_id& item_to_advertise : inventory_to_advertise) { - if (peer->inventory_advertised_to_peer.find(item_to_advertise) == peer->inventory_advertised_to_peer.end() ) + if (peer->inventory_advertised_to_peer.find(item_to_advertise) != peer->inventory_advertised_to_peer.end() ) wdump((*peer->inventory_advertised_to_peer.find(item_to_advertise))); - if (peer->inventory_peer_advertised_to_us.find(item_to_advertise) == peer->inventory_peer_advertised_to_us.end() ) + if (peer->inventory_peer_advertised_to_us.find(item_to_advertise) != peer->inventory_peer_advertised_to_us.end() ) wdump((*peer->inventory_peer_advertised_to_us.find(item_to_advertise))); if (peer->inventory_advertised_to_peer.find(item_to_advertise) == peer->inventory_advertised_to_peer.end() && From b84154d6e7f758c9eff348417a2c3414fe48e9cf Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Mon, 7 Sep 2015 12:52:09 -0400 Subject: [PATCH 04/18] wallet.cpp: Call update_account() to sync all account states to the blockchain when loading a wallet --- libraries/wallet/wallet.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/libraries/wallet/wallet.cpp b/libraries/wallet/wallet.cpp index 09d7ee29..575db736 100644 --- a/libraries/wallet/wallet.cpp +++ b/libraries/wallet/wallet.cpp @@ -648,6 +648,17 @@ public: FC_THROW( "Wallet chain ID does not match", ("wallet.chain_id", _wallet.chain_id) ("chain_id", _chain_id) ); + + vector< account_id_type > my_account_ids; + my_account_ids.reserve( _wallet.my_accounts.size() ); + for( const account_object& acct : _wallet.my_accounts ) + my_account_ids.push_back( acct.id ); + // TODO: Batch requests using _remote_db->get_accounts() + // to reduce number of round-trips. Remember get_accounts() has + // a limit of 100 results per call! + for( const account_id_type& acct_id : my_account_ids ) + _wallet.update_account( get_account( acct_id ) ); + return true; } void save_wallet_file(string wallet_filename = "") From 6c730462815d2364088fe32ddaad5d2aa8b7e34e Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Tue, 8 Sep 2015 16:03:53 -0400 Subject: [PATCH 05/18] block_tests.cpp: Implement transaction_expires_in_cache test #264 #299 --- tests/tests/block_tests.cpp | 136 ++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp index 2504e7ce..13c1b154 100644 --- a/tests/tests/block_tests.cpp +++ b/tests/tests/block_tests.cpp @@ -1018,4 +1018,140 @@ BOOST_FIXTURE_TEST_CASE( rsf_missed_blocks, database_fixture ) FC_LOG_AND_RETHROW() } +BOOST_FIXTURE_TEST_CASE( transaction_invalidated_in_cache, database_fixture ) +{ + try + { + ACTORS( (alice)(bob) ); + + auto generate_block = [&]( database& d ) -> signed_block + { + return d.generate_block(d.get_slot_time(1), d.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + }; + + wdump( (db.fetch_block_by_number(1)) ); + wdump( (db.fetch_block_by_id( db.head_block_id() ) ) ); + + signed_block b1 = generate_block(db); + wdump( (db.fetch_block_by_number(1)) ); + wdump( (db.fetch_block_by_id( db.head_block_id() ) ) ); + + fc::temp_directory data_dir2( graphene::utilities::temp_directory_path() ); + + database db2; + db2.open(data_dir2.path(), make_genesis); + BOOST_CHECK( db.get_chain_id() == db2.get_chain_id() ); + + while( db2.head_block_num() < db.head_block_num() ) + { + wdump( (db.head_block_num()) (db2.head_block_num()) ); + optional< signed_block > b = db.fetch_block_by_number( db2.head_block_num()+1 ); + db2.push_block(*b, database::skip_witness_signature); + } + wlog("caught up db2 to db"); + BOOST_CHECK( db2.get( alice_id ).name == "alice" ); + BOOST_CHECK( db2.get( bob_id ).name == "bob" ); + + db2.push_block(generate_block(db)); + transfer( account_id_type(), alice_id, asset( 1000 ) ); + transfer( account_id_type(), bob_id, asset( 1000 ) ); + db2.push_block(generate_block(db)); + + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 1000); + BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 1000); + BOOST_CHECK_EQUAL(db2.get_balance(alice_id, asset_id_type()).amount.value, 1000); + BOOST_CHECK_EQUAL(db2.get_balance( bob_id, asset_id_type()).amount.value, 1000); + + auto generate_and_send = [&]( int n ) + { + for( int i=0; i signed_transaction + { + signed_transaction tx; + transfer_operation xfer_op; + xfer_op.from = from; + xfer_op.to = to; + xfer_op.amount = asset( amount, asset_id_type() ); + xfer_op.fee = asset( 0, asset_id_type() ); + tx.operations.push_back( xfer_op ); + tx.set_expiration( db.head_block_time() + fc::seconds( 1000 ) ); + if( from == alice_id ) + sign( tx, alice_private_key ); + else + sign( tx, bob_private_key ); + return tx; + }; + + signed_transaction tx = generate_xfer_tx( alice_id, bob_id, 1000); + // put the tx in db tx cache + PUSH_TX( db, tx ); + // generate some blocks with db2, TODO: make tx expire in db's cache + generate_and_send(3); + + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 1000); + BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 1000); + + // generate a block with db and ensure we don't somehow apply it + PUSH_BLOCK(db2, generate_block(db)); + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 1000); + BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 1000); + + // now the tricky part... + // (A) Bob sends 1000 to Alice + // (B) Alice sends 2000 to Bob + // (C) Alice sends 500 to Bob + // + // We push AB, then receive a block containing C. + // we need to apply the block, then invalidate B in the cache. + // AB results in Alice having 0, Bob having 2000. + // C results in Alice having 500, Bob having 1500. + // + // This needs to occur while switching to a fork. + // + + signed_transaction tx_a = generate_xfer_tx( bob_id, alice_id, 1000 ); + signed_transaction tx_b = generate_xfer_tx( alice_id, bob_id, 2000 ); + signed_transaction tx_c = generate_xfer_tx( alice_id, bob_id, 500 ); + + generate_block( db ); + + PUSH_TX( db, tx_a ); + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 2000); + BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 0); + + PUSH_TX( db, tx_b ); + PUSH_TX( db2, tx_c ); + + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 0); + BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 2000); + + BOOST_CHECK_EQUAL(db2.get_balance(alice_id, asset_id_type()).amount.value, 500); + BOOST_CHECK_EQUAL(db2.get_balance( bob_id, asset_id_type()).amount.value, 1500); + + // generate enough blocks on db2 to cause db to switch forks + generate_and_send(2); + + // ensure both now reflect db2's view of the world + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 500); + BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 1500); + + BOOST_CHECK_EQUAL(db2.get_balance(alice_id, asset_id_type()).amount.value, 500); + BOOST_CHECK_EQUAL(db2.get_balance( bob_id, asset_id_type()).amount.value, 1500); + + // make sure we can still send blocks + generate_and_send(1); + } + catch (fc::exception& e) + { + edump((e.to_detail_string())); + throw; + } +} + BOOST_AUTO_TEST_SUITE_END() From 73174656332e5107a70c366935df33163406aa78 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Tue, 8 Sep 2015 16:31:52 -0400 Subject: [PATCH 06/18] block_tests.cpp: Fix #300 by generating more blocks in generate_empty_blocks() --- tests/tests/block_tests.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp index 13c1b154..68ee7fae 100644 --- a/tests/tests/block_tests.cpp +++ b/tests/tests/block_tests.cpp @@ -127,12 +127,14 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) // TODO: Don't generate this here auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); + signed_block b200; { database db; db.open(data_dir.path(), make_genesis ); b = db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); - for( uint32_t i = 1; i < 200; ++i ) + // n.b. we generate GRAPHENE_MIN_UNDO_HISTORY+1 extra blocks which will be discarded on save + for( uint32_t i = 1; i < 200+GRAPHENE_MIN_UNDO_HISTORY+1; ++i ) { BOOST_CHECK( db.head_block_id() == b.id() ); witness_id_type prev_witness = b.witness; @@ -140,6 +142,8 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) BOOST_CHECK( cur_witness != prev_witness ); b = db.generate_block(db.get_slot_time(1), cur_witness, init_account_priv_key, database::skip_nothing); BOOST_CHECK( b.witness == cur_witness ); + if( i == 199 ) + b200 = b; } db.close(); } @@ -147,6 +151,7 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) database db; db.open(data_dir.path(), []{return genesis_state_type();}); BOOST_CHECK_EQUAL( db.head_block_num(), 200 ); + b = b200; for( uint32_t i = 0; i < 200; ++i ) { BOOST_CHECK( db.head_block_id() == b.id() ); From 15ec55e52d76db9fc905e9d95c17df4876879f68 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Wed, 9 Sep 2015 10:46:13 -0400 Subject: [PATCH 07/18] block_tests.cpp: Fix transaction_invalidated_in_cache test, including transaction expiration --- tests/tests/block_tests.cpp | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp index 68ee7fae..4a904390 100644 --- a/tests/tests/block_tests.cpp +++ b/tests/tests/block_tests.cpp @@ -1076,7 +1076,7 @@ BOOST_FIXTURE_TEST_CASE( transaction_invalidated_in_cache, database_fixture ) } }; - auto generate_xfer_tx = [&]( account_id_type from, account_id_type to, share_type amount ) -> signed_transaction + auto generate_xfer_tx = [&]( account_id_type from, account_id_type to, share_type amount, int blocks_to_expire=10 ) -> signed_transaction { signed_transaction tx; transfer_operation xfer_op; @@ -1085,7 +1085,7 @@ BOOST_FIXTURE_TEST_CASE( transaction_invalidated_in_cache, database_fixture ) xfer_op.amount = asset( amount, asset_id_type() ); xfer_op.fee = asset( 0, asset_id_type() ); tx.operations.push_back( xfer_op ); - tx.set_expiration( db.head_block_time() + fc::seconds( 1000 ) ); + tx.set_expiration( db.head_block_time() + blocks_to_expire * db.get_global_properties().parameters.block_interval ); if( from == alice_id ) sign( tx, alice_private_key ); else @@ -1093,10 +1093,17 @@ BOOST_FIXTURE_TEST_CASE( transaction_invalidated_in_cache, database_fixture ) return tx; }; - signed_transaction tx = generate_xfer_tx( alice_id, bob_id, 1000); + signed_transaction tx = generate_xfer_tx( alice_id, bob_id, 1000, 2 ); + tx.set_expiration( db.head_block_time() + 2 * db.get_global_properties().parameters.block_interval ); + tx.signatures.clear(); + sign( tx, alice_private_key ); // put the tx in db tx cache PUSH_TX( db, tx ); - // generate some blocks with db2, TODO: make tx expire in db's cache + + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 0); + BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 2000); + + // generate some blocks with db2, make tx expire in db's cache generate_and_send(3); BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 1000); @@ -1120,7 +1127,7 @@ BOOST_FIXTURE_TEST_CASE( transaction_invalidated_in_cache, database_fixture ) // This needs to occur while switching to a fork. // - signed_transaction tx_a = generate_xfer_tx( bob_id, alice_id, 1000 ); + signed_transaction tx_a = generate_xfer_tx( bob_id, alice_id, 1000, 3 ); signed_transaction tx_b = generate_xfer_tx( alice_id, bob_id, 2000 ); signed_transaction tx_c = generate_xfer_tx( alice_id, bob_id, 500 ); @@ -1142,14 +1149,24 @@ BOOST_FIXTURE_TEST_CASE( transaction_invalidated_in_cache, database_fixture ) // generate enough blocks on db2 to cause db to switch forks generate_and_send(2); - // ensure both now reflect db2's view of the world + // db should invalidate B, but still be applying A, so the states don't agree + + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 1500); + BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 500); + + BOOST_CHECK_EQUAL(db2.get_balance(alice_id, asset_id_type()).amount.value, 500); + BOOST_CHECK_EQUAL(db2.get_balance( bob_id, asset_id_type()).amount.value, 1500); + + // This will cause A to expire in db + generate_and_send(1); + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 500); BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 1500); BOOST_CHECK_EQUAL(db2.get_balance(alice_id, asset_id_type()).amount.value, 500); BOOST_CHECK_EQUAL(db2.get_balance( bob_id, asset_id_type()).amount.value, 1500); - // make sure we can still send blocks + // Make sure we can generate and accept a plain old empty block on top of all this! generate_and_send(1); } catch (fc::exception& e) From ff2db08475908fad5e36df23d6c50256b9ab13f7 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Wed, 9 Sep 2015 11:11:58 -0400 Subject: [PATCH 08/18] database: Undo and re-apply pending_transactions before/after pushing a block. Fixes #299 --- libraries/chain/db_block.cpp | 22 +-- libraries/chain/db_update.cpp | 8 +- .../chain/include/graphene/chain/database.hpp | 42 ------ .../chain/include/graphene/chain/db_with.hpp | 126 ++++++++++++++++++ 4 files changed, 139 insertions(+), 59 deletions(-) create mode 100644 libraries/chain/include/graphene/chain/db_with.hpp diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp index 5985e0fa..7bed189d 100644 --- a/libraries/chain/db_block.cpp +++ b/libraries/chain/db_block.cpp @@ -17,6 +17,7 @@ */ #include +#include #include #include @@ -85,9 +86,13 @@ bool database::push_block(const signed_block& new_block, uint32_t skip) { idump((new_block.block_num())(new_block.id())); bool result; - with_skip_flags(skip, [&]() + detail::with_skip_flags( *this, skip, [&]() { - result = _push_block(new_block); + detail::without_pending_transactions( *this, std::move(_pending_block.transactions), + [&]() + { + result = _push_block(new_block); + }); }); return result; } @@ -156,11 +161,6 @@ bool database::_push_block(const signed_block& new_block) } } - // If there is a pending block session, then the database state is dirty with pending transactions. - // Drop the pending session to reset the database to a clean head block state. - // TODO: Preserve pending transactions, and re-apply any which weren't included in the new block. - clear_pending(); - try { auto session = _undo_db.start_undo_session(); apply_block(new_block, skip); @@ -187,7 +187,7 @@ bool database::_push_block(const signed_block& new_block) processed_transaction database::push_transaction( const signed_transaction& trx, uint32_t skip ) { try { processed_transaction result; - with_skip_flags( skip, [&]() + detail::with_skip_flags( *this, skip, [&]() { result = _push_transaction( trx ); } ); @@ -249,7 +249,7 @@ signed_block database::generate_block( ) { signed_block result; - with_skip_flags( skip, [&]() + detail::with_skip_flags( *this, skip, [&]() { result = _generate_block( when, witness_id, block_signing_private_key, true ); } ); @@ -390,7 +390,7 @@ void database::apply_block( const signed_block& next_block, uint32_t skip ) skip = ~0;// WE CAN SKIP ALMOST EVERYTHING } - with_skip_flags( skip, [&]() + detail::with_skip_flags( *this, skip, [&]() { _apply_block( next_block ); } ); @@ -479,7 +479,7 @@ void database::notify_changed_objects() processed_transaction database::apply_transaction(const signed_transaction& trx, uint32_t skip) { processed_transaction result; - with_skip_flags(skip, [&]() + detail::with_skip_flags( *this, skip, [&]() { result = _apply_transaction(trx); }); diff --git a/libraries/chain/db_update.cpp b/libraries/chain/db_update.cpp index 11e5c0e0..cccc3bb2 100644 --- a/libraries/chain/db_update.cpp +++ b/libraries/chain/db_update.cpp @@ -17,6 +17,7 @@ */ #include +#include #include #include @@ -100,10 +101,6 @@ void database::update_pending_block(const signed_block& next_block, uint8_t curr { _pending_block.timestamp = next_block.timestamp + current_block_interval; _pending_block.previous = next_block.id(); - auto old_pending_trx = std::move(_pending_block.transactions); - _pending_block.transactions.clear(); - for( auto old_trx : old_pending_trx ) - push_transaction( old_trx ); } void database::clear_expired_transactions() @@ -142,7 +139,7 @@ void database::clear_expired_proposals() void database::clear_expired_orders() { - with_skip_flags( + detail::with_skip_flags( *this, get_node_properties().skip_flags | skip_authority_check, [&](){ transaction_evaluation_state cancel_context(this); @@ -158,7 +155,6 @@ void database::clear_expired_orders() } }); - //Process expired force settlement orders auto& settlement_index = get_index_type().indices().get(); if( !settlement_index.empty() ) diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp index a601b49c..abfd1936 100644 --- a/libraries/chain/include/graphene/chain/database.hpp +++ b/libraries/chain/include/graphene/chain/database.hpp @@ -40,31 +40,6 @@ namespace graphene { namespace chain { using graphene::db::abstract_object; using graphene::db::object; - namespace detail - { - /** - * Class used to help the with_skip_flags implementation. - * It must be defined in this header because it must be - * available to the with_skip_flags implementation, - * which is a template and therefore must also be defined - * in this header. - */ - struct skip_flags_restorer - { - skip_flags_restorer( node_property_object& npo, uint32_t old_skip_flags ) - : _npo( npo ), _old_skip_flags( old_skip_flags ) - {} - - ~skip_flags_restorer() - { - _npo.skip_flags = _old_skip_flags; - } - - node_property_object& _npo; - uint32_t _old_skip_flags; - }; - } - /** * @class database * @brief tracks the blockchain state in an extensible manner @@ -270,23 +245,6 @@ namespace graphene { namespace chain { node_property_object& node_properties(); - /** - * Set the skip_flags to the given value, call callback, - * then reset skip_flags to their previous value after - * callback is done. - */ - template< typename Lambda > - void with_skip_flags( - uint32_t skip_flags, - Lambda callback ) - { - node_property_object& npo = node_properties(); - detail::skip_flags_restorer restorer( npo, npo.skip_flags ); - npo.skip_flags = skip_flags; - callback(); - return; - } - //////////////////// db_init.cpp //////////////////// void initialize_evaluators(); diff --git a/libraries/chain/include/graphene/chain/db_with.hpp b/libraries/chain/include/graphene/chain/db_with.hpp new file mode 100644 index 00000000..09781f0f --- /dev/null +++ b/libraries/chain/include/graphene/chain/db_with.hpp @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2015, Cryptonomex, Inc. + * All rights reserved. + * + * This source code is provided for evaluation in private test networks only, until September 8, 2015. After this date, this license expires and + * the code may not be used, modified or distributed for any purpose. Redistribution and use in source and binary forms, with or without modification, + * are permitted until September 8, 2015, provided that the following conditions are met: + * + * 1. The code and/or derivative works are used only for private test networks consisting of no more than 10 P2P nodes. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#pragma once + +#include + +/* + * This file provides with() functions which modify the database + * temporarily, then restore it. These functions are mostly internal + * implementation detail of the database. + * + * Essentially, we want to be able to use "finally" to restore the + * database regardless of whether an exception is thrown or not, but there + * is no "finally" in C++. Instead, C++ requires us to create a struct + * and put the finally block in a destructor. Aagh! + */ + +namespace graphene { namespace chain { namespace detail { +/** + * Class used to help the with_skip_flags implementation. + * It must be defined in this header because it must be + * available to the with_skip_flags implementation, + * which is a template and therefore must also be defined + * in this header. + */ +struct skip_flags_restorer +{ + skip_flags_restorer( node_property_object& npo, uint32_t old_skip_flags ) + : _npo( npo ), _old_skip_flags( old_skip_flags ) + {} + + ~skip_flags_restorer() + { + _npo.skip_flags = _old_skip_flags; + } + + node_property_object& _npo; + uint32_t _old_skip_flags; +}; + +/** + * Class used to help the without_pending_transactions + * implementation. + */ +struct pending_transactions_restorer +{ + pending_transactions_restorer( database& db, std::vector&& pending_transactions ) + : _db(db), _pending_transactions( std::move(pending_transactions) ) + { + _db.clear_pending(); + } + + ~pending_transactions_restorer() + { + for( const processed_transaction& tx : _pending_transactions ) + { + try + { + // since push_transaction() takes a signed_transaction, + // the operation_results field will be ignored. + _db.push_transaction( tx ); + } + catch( const fc::exception& e ) + { + wlog( "Pending transaction became invalid after switching to block ${b}", ("b", _db.head_block_id()) ); + wlog( "The invalid pending transaction is ${t}", ("t", tx) ); + wlog( "The invalid pending transaction caused exception ${e}", ("e", e) ); + } + } + } + + database& _db; + std::vector< processed_transaction > _pending_transactions; +}; + +/** + * Set the skip_flags to the given value, call callback, + * then reset skip_flags to their previous value after + * callback is done. + */ +template< typename Lambda > +void with_skip_flags( + database& db, + uint32_t skip_flags, + Lambda callback ) +{ + node_property_object& npo = db.node_properties(); + skip_flags_restorer restorer( npo, npo.skip_flags ); + npo.skip_flags = skip_flags; + callback(); + return; +} + +/** + * Empty pending_transactions, call callback, + * then reset pending_transactions after callback is done. + * + * Pending transactions which no longer validate will be culled. + */ +template< typename Lambda > +void without_pending_transactions( + database& db, + std::vector&& pending_transactions, + Lambda callback ) +{ + pending_transactions_restorer restorer( db, std::move(pending_transactions) ); + callback(); + return; +} + +} } } // graphene::chain::detail From 59850cee986cbabcc8255a9ca987e5f73ccf71d5 Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Wed, 9 Sep 2015 13:59:43 -0400 Subject: [PATCH 09/18] Fix #305 - replay blockchain works again --- libraries/chain/db_management.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp index 4a5cf899..baf4548b 100644 --- a/libraries/chain/db_management.cpp +++ b/libraries/chain/db_management.cpp @@ -101,9 +101,12 @@ void database::open( { _fork_db.start_block( *last_block ); idump((last_block->id())(last_block->block_num())); + if( last_block->id() != head_block_id() ) + { + FC_ASSERT( head_block_num() == 0, "last block ID does not match current chain state" ); + } } idump((head_block_id())(head_block_num())); - FC_ASSERT( !last_block || last_block->id() == head_block_id() ); } FC_CAPTURE_AND_RETHROW( (data_dir) ) } From 70d3f36fba9ff6ac466f117f7fe241d3584029c3 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Wed, 9 Sep 2015 14:44:31 -0400 Subject: [PATCH 10/18] database: Handle gaps in block database when reindexing --- libraries/chain/block_database.cpp | 36 +++++++++++++++++++ libraries/chain/db_management.cpp | 32 +++++++++++++---- .../include/graphene/chain/block_database.hpp | 1 + 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/libraries/chain/block_database.cpp b/libraries/chain/block_database.cpp index c0005971..281dd387 100644 --- a/libraries/chain/block_database.cpp +++ b/libraries/chain/block_database.cpp @@ -237,4 +237,40 @@ optional block_database::last()const } return optional(); } + +optional block_database::last_id()const +{ + try + { + index_entry e; + _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); + + if( _block_num_to_pos.tellp() < sizeof(index_entry) ) + return optional(); + + _block_num_to_pos.seekg( -sizeof(index_entry), _block_num_to_pos.end ); + _block_num_to_pos.read( (char*)&e, sizeof(e) ); + uint64_t pos = _block_num_to_pos.tellg(); + while( e.block_size == 0 && pos > 0 ) + { + pos -= sizeof(index_entry); + _block_num_to_pos.seekg( pos ); + _block_num_to_pos.read( (char*)&e, sizeof(e) ); + } + + if( e.block_size == 0 ) + return optional(); + + return e.block_id; + } + catch (const fc::exception&) + { + } + catch (const std::exception&) + { + } + return optional(); +} + + } } diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp index baf4548b..c33b3ad1 100644 --- a/libraries/chain/db_management.cpp +++ b/libraries/chain/db_management.cpp @@ -58,12 +58,32 @@ void database::reindex(fc::path data_dir, const genesis_state_type& initial_allo for( uint32_t i = 1; i <= last_block_num; ++i ) { if( i % 2000 == 0 ) std::cerr << " " << double(i*100)/last_block_num << "% "< block = _block_id_to_block.fetch_by_number(i); + if( !block.valid() ) + { + wlog( "Reindexing terminated due to gap: Block ${i} does not exist!", ("i", i) ); + uint32_t dropped_count = 0; + while( true ) + { + fc::optional< block_id_type > last_id = _block_id_to_block.last_id(); + // this can trigger if we attempt to e.g. read a file that has block #2 but no block #1 + if( !last_id.valid() ) + break; + // we've caught up to the gap + if( block_header::num_from_id( *last_id ) <= i ) + break; + _block_id_to_block.remove( *last_id ); + dropped_count++; + } + wlog( "Dropped ${n} blocks from after the gap", ("n", dropped_count) ); + break; + } + apply_block(*block, skip_witness_signature | + skip_transaction_signatures | + skip_transaction_dupe_check | + skip_tapos_check | + skip_witness_schedule_check | + skip_authority_check); } _undo_db.enable(); auto end = fc::time_point::now(); diff --git a/libraries/chain/include/graphene/chain/block_database.hpp b/libraries/chain/include/graphene/chain/block_database.hpp index 1e8a97a6..816df798 100644 --- a/libraries/chain/include/graphene/chain/block_database.hpp +++ b/libraries/chain/include/graphene/chain/block_database.hpp @@ -36,6 +36,7 @@ namespace graphene { namespace chain { optional fetch_optional( const block_id_type& id )const; optional fetch_by_number( uint32_t block_num )const; optional last()const; + optional last_id()const; private: mutable std::fstream _blocks; mutable std::fstream _block_num_to_pos; From 09c623b3f929e1b8495e8adf2cdd1012be738919 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Wed, 9 Sep 2015 16:03:27 -0400 Subject: [PATCH 11/18] Remove maximum_expiration parameter #308 --- libraries/chain/db_update.cpp | 4 +--- libraries/chain/get_config.cpp | 1 - libraries/chain/include/graphene/chain/config.hpp | 1 - .../include/graphene/chain/protocol/chain_parameters.hpp | 2 +- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/libraries/chain/db_update.cpp b/libraries/chain/db_update.cpp index cccc3bb2..debb05a0 100644 --- a/libraries/chain/db_update.cpp +++ b/libraries/chain/db_update.cpp @@ -109,9 +109,7 @@ void database::clear_expired_transactions() //Transactions must have expired by at least two forking windows in order to be removed. auto& transaction_idx = static_cast(get_mutable_index(implementation_ids, impl_transaction_object_type)); const auto& dedupe_index = transaction_idx.indices().get(); - const auto& global_parameters = get_global_properties().parameters; - while( !dedupe_index.empty() - && head_block_time() - dedupe_index.rbegin()->trx.expiration >= fc::seconds(global_parameters.maximum_expiration) ) + while( (!dedupe_index.empty()) && (head_block_time() > dedupe_index.rbegin()->trx.expiration) ) transaction_idx.remove(*dedupe_index.rbegin()); } diff --git a/libraries/chain/get_config.cpp b/libraries/chain/get_config.cpp index 9f10ce78..1d81ecfb 100644 --- a/libraries/chain/get_config.cpp +++ b/libraries/chain/get_config.cpp @@ -45,7 +45,6 @@ fc::variant_object get_config() result[ "GRAPHENE_DEFAULT_MAX_TIME_UNTIL_EXPIRATION" ] = GRAPHENE_DEFAULT_MAX_TIME_UNTIL_EXPIRATION; result[ "GRAPHENE_DEFAULT_MAINTENANCE_INTERVAL" ] = GRAPHENE_DEFAULT_MAINTENANCE_INTERVAL; result[ "GRAPHENE_DEFAULT_MAINTENANCE_SKIP_SLOTS" ] = GRAPHENE_DEFAULT_MAINTENANCE_SKIP_SLOTS; - result[ "GRAPHENE_DEFAULT_MAX_EXPIRATION_SEC" ] = GRAPHENE_DEFAULT_MAX_EXPIRATION_SEC; result[ "GRAPHENE_MIN_UNDO_HISTORY" ] = GRAPHENE_MIN_UNDO_HISTORY; result[ "GRAPHENE_MAX_UNDO_HISTORY" ] = GRAPHENE_MAX_UNDO_HISTORY; result[ "GRAPHENE_MIN_BLOCK_SIZE_LIMIT" ] = GRAPHENE_MIN_BLOCK_SIZE_LIMIT; diff --git a/libraries/chain/include/graphene/chain/config.hpp b/libraries/chain/include/graphene/chain/config.hpp index 7256fbb2..3ad559ea 100644 --- a/libraries/chain/include/graphene/chain/config.hpp +++ b/libraries/chain/include/graphene/chain/config.hpp @@ -45,7 +45,6 @@ #define GRAPHENE_DEFAULT_MAX_TIME_UNTIL_EXPIRATION (60*60*24) // seconds, aka: 1 day #define GRAPHENE_DEFAULT_MAINTENANCE_INTERVAL (60*60*24) // seconds, aka: 1 day #define GRAPHENE_DEFAULT_MAINTENANCE_SKIP_SLOTS 3 // number of slots to skip for maintenance interval -#define GRAPHENE_DEFAULT_MAX_EXPIRATION_SEC (60*60) // 1 hour #define GRAPHENE_MIN_UNDO_HISTORY 10 #define GRAPHENE_MAX_UNDO_HISTORY 1000 diff --git a/libraries/chain/include/graphene/chain/protocol/chain_parameters.hpp b/libraries/chain/include/graphene/chain/protocol/chain_parameters.hpp index ac91d777..1f0e8d76 100644 --- a/libraries/chain/include/graphene/chain/protocol/chain_parameters.hpp +++ b/libraries/chain/include/graphene/chain/protocol/chain_parameters.hpp @@ -41,7 +41,7 @@ namespace graphene { namespace chain { uint32_t committee_proposal_review_period = GRAPHENE_DEFAULT_COMMITTEE_PROPOSAL_REVIEW_PERIOD_SEC; ///< minimum time in seconds that a proposed transaction requiring committee authority may not be signed, prior to expiration uint32_t maximum_transaction_size = GRAPHENE_DEFAULT_MAX_TRANSACTION_SIZE; ///< maximum allowable size in bytes for a transaction uint32_t maximum_block_size = GRAPHENE_DEFAULT_MAX_BLOCK_SIZE; ///< maximum allowable size in bytes for a block - uint32_t maximum_expiration = GRAPHENE_DEFAULT_MAX_EXPIRATION_SEC; ///< maximum number of seconds in the future a transaction may expire + uint32_t maximum_expiration = 0; ///< ignored, but included to ensure we don't hardfork; see #308 uint32_t maximum_time_until_expiration = GRAPHENE_DEFAULT_MAX_TIME_UNTIL_EXPIRATION; ///< maximum lifetime in seconds for transactions to be valid, before expiring uint32_t maximum_proposal_lifetime = GRAPHENE_DEFAULT_MAX_PROPOSAL_LIFETIME_SEC; ///< maximum lifetime in seconds for proposed transactions to be kept, before expiring uint8_t maximum_asset_whitelist_authorities = GRAPHENE_DEFAULT_MAX_ASSET_WHITELIST_AUTHORITIES; ///< maximum number of accounts which an asset may list as authorities for its whitelist OR blacklist From b8b2bcf97e51ee24fe15864259fc61be2244a9f0 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Wed, 9 Sep 2015 17:36:36 -0400 Subject: [PATCH 12/18] Implement --dbg-init-key command line option to take control of init witnesses for debugging #307 --- libraries/app/application.cpp | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index c9783d53..a55db76e 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -219,6 +219,15 @@ namespace detail { fc::remove_all(_data_dir / "blockchain/dblock"); } + void set_dbg_init_key( genesis_state_type& genesis, const std::string& init_key ) + { + flat_set< std::string > initial_witness_names; + public_key_type init_pubkey( init_key ); + for( uint64_t i=0; icount("genesis-json") ) { genesis_state_type genesis = fc::json::from_file(_options->at("genesis-json").as()).as(); + bool modified_genesis = false; if( _options->count("genesis-timestamp") ) { genesis.initial_timestamp = fc::time_point_sec( graphene::time::now() ) + genesis.initial_parameters.block_interval + _options->at("genesis-timestamp").as(); genesis.initial_timestamp -= genesis.initial_timestamp.sec_since_epoch() % genesis.initial_parameters.block_interval; + modified_genesis = true; std::cerr << "Used genesis timestamp: " << genesis.initial_timestamp.to_iso_string() << " (PLEASE RECORD THIS)\n"; } + if( _options->count("dbg-init-key") ) + { + std::string init_key = _options->at( "dbg-init-key" ).as(); + FC_ASSERT( genesis.initial_witness_candidates.size() >= genesis.initial_active_witnesses ); + set_dbg_init_key( genesis, init_key ); + modified_genesis = true; + std::cerr << "Set init witness key to " << init_key << "\n"; + } + if( modified_genesis ) + { + std::cerr << "WARNING: GENESIS WAS MODIFIED, YOUR CHAIN ID MAY BE DIFFERENT\n"; + } return genesis; } else @@ -610,6 +633,7 @@ void application::set_program_options(boost::program_options::options_descriptio ("server-pem,p", bpo::value()->implicit_value("server.pem"), "The TLS certificate file for this server") ("server-pem-password,P", bpo::value()->implicit_value(""), "Password for this certificate") ("genesis-json", bpo::value(), "File to read Genesis State from") + ("dbg-init-key", bpo::value(), "Block signing key to use for init witnesses, overrides genesis file") ("api-access", bpo::value(), "JSON file specifying API permissions") ; command_line_options.add(configuration_file_options); From 9991a4117fc4c2fc6109f5b552ec300c4e94fe76 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Thu, 10 Sep 2015 09:59:36 -0400 Subject: [PATCH 13/18] README.md: Update/move instructions for running private testnet --- README.md | 24 +----------------------- docs | 2 +- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 0fa53eb2..50d9c1d3 100644 --- a/README.md +++ b/README.md @@ -165,29 +165,7 @@ it is fairly simple to write API methods to expose database methods. Running private testnet ----------------------- -Normally `witness_node` assumes it won't be producing blocks from -genesis, or against very old chain state. We need to get `witness_node` -to discard this assumption if we actually want to start a new chain, -so we will need to specify in `config.ini`: - - enable-stale-production = true - -We also need to specify which witnesses will produce blocks locally; -`witness_node` does not assume that it should produce blocks for a given -witness just because it has the correct private key to do so. There are -ten witnesses at genesis of the testnet, block production can be -enabled for all of them by specifying multiple times in `config.ini`: - - witness-id = "1.6.0" - witness-id = "1.6.1" - witness-id = "1.6.2" - witness-id = "1.6.3" - witness-id = "1.6.4" - witness-id = "1.6.5" - witness-id = "1.6.6" - witness-id = "1.6.7" - witness-id = "1.6.8" - witness-id = "1.6.9" +See the [documentation](docs/private-testnet.md) if you want to run a private testnet. Questions --------- diff --git a/docs b/docs index cdc8ea81..c004ae42 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit cdc8ea8133a999afef8051700a4ce8edb0988ec4 +Subproject commit c004ae42a72d86bbc6c7e8d065deed284fd093a5 From 65165c416fb8bbe2d6cc7d542047000e710cf279 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Thu, 10 Sep 2015 10:01:11 -0400 Subject: [PATCH 14/18] README.md: Fix URL in previous commit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 50d9c1d3..fa374b92 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ it is fairly simple to write API methods to expose database methods. Running private testnet ----------------------- -See the [documentation](docs/private-testnet.md) if you want to run a private testnet. +See the [documentation](https://github.com/cryptonomex/graphene/wiki/private-testnet) if you want to run a private testnet. Questions --------- From 0b6f7fe430ce1ff15e01d76ea4762dbf0700e952 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Thu, 10 Sep 2015 11:16:23 -0400 Subject: [PATCH 15/18] wallet.cpp: Ask remote DB for accounts on startup --- libraries/wallet/wallet.cpp | 52 ++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/libraries/wallet/wallet.cpp b/libraries/wallet/wallet.cpp index 575db736..7a0608b7 100644 --- a/libraries/wallet/wallet.cpp +++ b/libraries/wallet/wallet.cpp @@ -649,15 +649,49 @@ public: ("wallet.chain_id", _wallet.chain_id) ("chain_id", _chain_id) ); - vector< account_id_type > my_account_ids; - my_account_ids.reserve( _wallet.my_accounts.size() ); - for( const account_object& acct : _wallet.my_accounts ) - my_account_ids.push_back( acct.id ); - // TODO: Batch requests using _remote_db->get_accounts() - // to reduce number of round-trips. Remember get_accounts() has - // a limit of 100 results per call! - for( const account_id_type& acct_id : my_account_ids ) - _wallet.update_account( get_account( acct_id ) ); + size_t account_pagination = 100; + vector< account_id_type > account_ids_to_send; + size_t n = _wallet.my_accounts.size(); + account_ids_to_send.reserve( std::min( account_pagination, n ) ); + auto it = _wallet.my_accounts.begin(); + + for( size_t start=0; start start ); + account_ids_to_send.clear(); + std::vector< account_object > old_accounts; + for( size_t i=start; i > accounts = _remote_db->get_accounts(account_ids_to_send); + // server response should be same length as request + FC_ASSERT( accounts.size() == account_ids_to_send.size() ); + size_t i = 0; + for( const optional< account_object >& acct : accounts ) + { + account_object& old_acct = old_accounts[i]; + if( !acct.valid() ) + { + elog( "Could not find account ${id} : \"${name}\" does not exist on the chain!", ("id", old_acct.id)("name", old_acct.name) ); + i++; + continue; + } + // this check makes sure the server didn't send results + // in a different order, or accounts we didn't request + FC_ASSERT( acct->id == old_acct.id ); + if( fc::json::to_string(*acct) != fc::json::to_string(old_acct) ) + { + wlog( "Account ${id} : \"${name}\" updated on chain", ("id", acct->id)("name", acct->name) ); + } + _wallet.update_account( *acct ); + i++; + } + } return true; } From e1530709d973c092526977c09cd00574ebba2f9e Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Thu, 10 Sep 2015 12:08:01 -0400 Subject: [PATCH 16/18] witness.cpp: Set skip_undo_history_check when --enable-stale-production is specified --- .../witness/include/graphene/witness/witness.hpp | 1 + libraries/plugins/witness/witness.cpp | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/libraries/plugins/witness/include/graphene/witness/witness.hpp b/libraries/plugins/witness/include/graphene/witness/witness.hpp index c82b83d8..df033094 100644 --- a/libraries/plugins/witness/include/graphene/witness/witness.hpp +++ b/libraries/plugins/witness/include/graphene/witness/witness.hpp @@ -75,6 +75,7 @@ private: bool _production_enabled = false; bool _consecutive_production_enabled = false; uint32_t _required_witness_participation = 33 * GRAPHENE_1_PERCENT; + uint32_t _production_skip_flags = graphene::chain::database::skip_nothing; std::map _private_keys; std::set _witnesses; diff --git a/libraries/plugins/witness/witness.cpp b/libraries/plugins/witness/witness.cpp index 24eb0309..db5154ec 100644 --- a/libraries/plugins/witness/witness.cpp +++ b/libraries/plugins/witness/witness.cpp @@ -123,8 +123,12 @@ void witness_plugin::plugin_startup() { ilog("Launching block production for ${n} witnesses.", ("n", _witnesses.size())); app().set_block_production(true); - if( _production_enabled && (d.head_block_num() == 0) ) - new_chain_banner(d); + if( _production_enabled ) + { + if( d.head_block_num() == 0 ) + new_chain_banner(d); + _production_skip_flags |= graphene::chain::database::skip_undo_history_check; + } schedule_production_loop(); } else elog("No witnesses configured! Please add witness IDs and private keys to configuration."); @@ -278,7 +282,7 @@ block_production_condition::block_production_condition_enum witness_plugin::mayb scheduled_time, scheduled_witness, private_key_itr->second, - graphene::chain::database::skip_nothing + _production_skip_flags ); capture("n", block.block_num())("t", block.timestamp)("c", now); p2p_node().broadcast(net::block_message(block)); From 77ac6eb689eeafad17aa1f59757977700614691e Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Thu, 10 Sep 2015 15:41:13 -0400 Subject: [PATCH 17/18] db_init.cpp: Don't allocate unnecessary asset_dynamic_data_object --- libraries/chain/db_init.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/chain/db_init.cpp b/libraries/chain/db_init.cpp index f3361bf1..55122d12 100644 --- a/libraries/chain/db_init.cpp +++ b/libraries/chain/db_init.cpp @@ -331,13 +331,13 @@ void database::init_genesis(const genesis_state_type& genesis_state) // Create more special assets while( true ) { + uint64_t id = get_index().get_next_id().instance(); + if( id >= genesis_state.immutable_parameters.num_special_assets ) + break; const asset_dynamic_data_object& dyn_asset = create([&](asset_dynamic_data_object& a) { a.current_supply = 0; }); - uint64_t id = get_index().get_next_id().instance(); - if( id >= genesis_state.immutable_parameters.num_special_assets ) - break; const asset_object& asset_obj = create( [&]( asset_object& a ) { a.symbol = "SPECIAL" + std::to_string( id ); a.options.max_supply = 0; From 358a88037c069b0b61ebdc0ac510e846b1fc9764 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Thu, 10 Sep 2015 15:41:49 -0400 Subject: [PATCH 18/18] wallet.cpp: Subscribe to block updates, fix #302 --- libraries/app/api.cpp | 10 +++++++++- libraries/app/include/graphene/app/api.hpp | 3 +++ libraries/wallet/wallet.cpp | 11 +++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index 5b61bc61..e440854f 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -923,10 +923,18 @@ namespace graphene { namespace app { */ void database_api::on_applied_block() { + if (_block_applied_callback) + { + auto capture_this = shared_from_this(); + block_id_type block_id = _db.head_block_id(); + fc::async([this,capture_this,block_id](){ + _block_applied_callback(fc::variant(block_id)); + }); + } + if(_market_subscriptions.size() == 0) return; - const auto& ops = _db.get_applied_operations(); map< std::pair, vector> > subscribed_markets_ops; for(const auto& op : ops) diff --git a/libraries/app/include/graphene/app/api.hpp b/libraries/app/include/graphene/app/api.hpp index ca83accd..8a94706f 100644 --- a/libraries/app/include/graphene/app/api.hpp +++ b/libraries/app/include/graphene/app/api.hpp @@ -367,6 +367,7 @@ namespace graphene { namespace app { void set_subscribe_callback( std::function cb, bool clear_filter ); void set_pending_transaction_callback( std::function cb ){ _pending_trx_callback = cb; } + void set_block_applied_callback( std::function cb ){ _block_applied_callback = cb; } private: template void subscribe_to_item( const T& i )const @@ -398,6 +399,7 @@ namespace graphene { namespace app { mutable fc::bloom_filter _subscribe_filter; std::function _subscribe_callback; std::function _pending_trx_callback; + std::function _block_applied_callback; boost::signals2::scoped_connection _change_connection; boost::signals2::scoped_connection _removed_connection; @@ -606,6 +608,7 @@ FC_API(graphene::app::database_api, (get_required_fees) (set_subscribe_callback) (set_pending_transaction_callback) + (set_block_applied_callback) (validate_transaction) ) FC_API(graphene::app::history_api, diff --git a/libraries/wallet/wallet.cpp b/libraries/wallet/wallet.cpp index 7a0608b7..7d5cef4b 100644 --- a/libraries/wallet/wallet.cpp +++ b/libraries/wallet/wallet.cpp @@ -375,6 +375,12 @@ public: ("chain_id", _chain_id) ); } init_prototype_ops(); + + _remote_db->set_block_applied_callback( [this](const variant& block_id ) + { + on_block_applied( block_id ); + } ); + _wallet.chain_id = _chain_id; _wallet.ws_server = initial_data.ws_server; _wallet.ws_user = initial_data.ws_user; @@ -408,6 +414,11 @@ public: } } + void on_block_applied( const variant& block_id ) + { + fc::async([this]{resync();}, "Resync after block"); + } + bool copy_wallet_file( string destination_filename ) { fc::path src_path = get_wallet_filename();