From a79eff2761119b34d9c8854c0a9ab9c2fafae16e Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Tue, 25 Aug 2015 13:45:20 -0400 Subject: [PATCH 1/8] progress toward witness schedule refactor --- libraries/CMakeLists.txt | 2 +- libraries/chain/db_block.cpp | 20 ------------------- libraries/chain/db_update.cpp | 8 +------- libraries/chain/db_witness_schedule.cpp | 6 +++--- .../graphene/chain/global_property_object.hpp | 10 +++++++--- .../include/graphene/chain/protocol/block.hpp | 5 +---- .../include/graphene/chain/witness_object.hpp | 5 +++-- libraries/wallet/wallet.cpp | 1 - programs/size_checker/main.cpp | 15 ++++++++++++++ 9 files changed, 31 insertions(+), 41 deletions(-) diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt index 2a0754ef..be71012d 100644 --- a/libraries/CMakeLists.txt +++ b/libraries/CMakeLists.txt @@ -4,7 +4,7 @@ add_subdirectory( deterministic_openssl_rand ) add_subdirectory( chain ) add_subdirectory( egenesis ) add_subdirectory( net ) -add_subdirectory( p2p ) +#add_subdirectory( p2p ) add_subdirectory( time ) add_subdirectory( utilities ) add_subdirectory( app ) diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp index 99131146..89dd4613 100644 --- a/libraries/chain/db_block.cpp +++ b/libraries/chain/db_block.cpp @@ -267,24 +267,6 @@ signed_block database::_generate_block( _pending_block.timestamp = when; - // Genesis witnesses start with a default initial secret - if( witness_obj.next_secret_hash == secret_hash_type::hash( secret_hash_type() ) ) - { - _pending_block.previous_secret = secret_hash_type(); - } - else - { - secret_hash_type::encoder last_enc; - fc::raw::pack( last_enc, block_signing_private_key ); - fc::raw::pack( last_enc, witness_obj.previous_secret ); - _pending_block.previous_secret = last_enc.result(); - } - - secret_hash_type::encoder next_enc; - fc::raw::pack( next_enc, block_signing_private_key ); - fc::raw::pack( next_enc, _pending_block.previous_secret ); - _pending_block.next_secret_hash = secret_hash_type::hash(next_enc.result()); - _pending_block.transaction_merkle_root = _pending_block.calculate_merkle_root(); _pending_block.witness = witness_id; @@ -555,8 +537,6 @@ const witness_object& database::validate_block_header( uint32_t skip, const sign FC_ASSERT( _pending_block.previous == next_block.previous, "", ("pending.prev",_pending_block.previous)("next.prev",next_block.previous) ); FC_ASSERT( _pending_block.timestamp <= next_block.timestamp, "", ("_pending_block.timestamp",_pending_block.timestamp)("next",next_block.timestamp)("blocknum",next_block.block_num()) ); const witness_object& witness = next_block.witness(*this); - FC_ASSERT( secret_hash_type::hash( next_block.previous_secret ) == witness.next_secret_hash, "", - ("previous_secret", next_block.previous_secret)("next_secret_hash", witness.next_secret_hash)); if( !(skip&skip_witness_signature) ) FC_ASSERT( next_block.validate_signee( witness.signing_key ) ); diff --git a/libraries/chain/db_update.cpp b/libraries/chain/db_update.cpp index ded6a04a..da6e84ef 100644 --- a/libraries/chain/db_update.cpp +++ b/libraries/chain/db_update.cpp @@ -44,11 +44,6 @@ void database::update_global_dynamic_data( const signed_block& b ) // dynamic global properties updating modify( _dgp, [&]( dynamic_global_property_object& dgp ){ - secret_hash_type::encoder enc; - fc::raw::pack( enc, dgp.random ); - fc::raw::pack( enc, b.previous_secret ); - dgp.random = enc.result(); - if( _checkpoints.size() && _checkpoints.rbegin()->first >= b.block_num() ) dgp.recently_missed_count = 0; else if( missed_blocks ) @@ -92,8 +87,7 @@ void database::update_signing_witness(const witness_object& signing_witness, con modify( signing_witness, [&]( witness_object& _wit ) { - _wit.previous_secret = new_block.previous_secret; - _wit.next_secret_hash = new_block.next_secret_hash; + _wit.last_slot_num = new_block.block_num(); /// TODO: plus total missed blocks } ); } diff --git a/libraries/chain/db_witness_schedule.cpp b/libraries/chain/db_witness_schedule.cpp index 2cb8b711..30389bc8 100644 --- a/libraries/chain/db_witness_schedule.cpp +++ b/libraries/chain/db_witness_schedule.cpp @@ -132,9 +132,8 @@ void database::update_witness_schedule(const signed_block& next_block) witness_id_type wit; - const dynamic_global_property_object& dpo = get_dynamic_global_properties(); + //const dynamic_global_property_object& dpo = get_dynamic_global_properties(); - assert( dpo.random.data_size() == witness_scheduler_rng::seed_length ); assert( witness_scheduler_rng::seed_length == wso.rng_seed.size() ); modify(wso, [&](witness_schedule_object& _wso) @@ -161,8 +160,9 @@ void database::update_witness_schedule(const signed_block& next_block) } while( !_wso.scheduler.get_slot(schedule_needs_filled, wit) ) { + auto random = fc::ripemd160::hash( next_block.timestamp ); if( _wso.scheduler.produce_schedule(rng) & emit_turn ) - memcpy(_wso.rng_seed.begin(), dpo.random.data(), dpo.random.data_size()); + memcpy(_wso.rng_seed.begin(), random.data(), random.data_size()); } _wso.last_scheduling_block = next_block.block_num(); _wso.recent_slots_filled = ( diff --git a/libraries/chain/include/graphene/chain/global_property_object.hpp b/libraries/chain/include/graphene/chain/global_property_object.hpp index dacb5ce2..82a5703b 100644 --- a/libraries/chain/include/graphene/chain/global_property_object.hpp +++ b/libraries/chain/include/graphene/chain/global_property_object.hpp @@ -42,7 +42,7 @@ namespace graphene { namespace chain { chain_parameters parameters; optional pending_parameters; - uint32_t next_available_vote_id = 0; + uint32_t next_available_vote_id = 0; vector active_committee_members; // updated once per maintenance interval flat_set active_witnesses; // updated once per maintenance interval // n.b. witness scheduling is done by witness_schedule object @@ -64,7 +64,6 @@ namespace graphene { namespace chain { static const uint8_t space_id = implementation_ids; static const uint8_t type_id = impl_dynamic_global_property_object_type; - secret_hash_type random; uint32_t head_block_number = 0; block_id_type head_block_id; time_point_sec time; @@ -81,6 +80,11 @@ namespace graphene { namespace chain { */ uint32_t recently_missed_count = 0; + /** this is the set of witnesses that may produce the next block because they + * haven't produced any blocks recently. + */ + vector potential_witnesses; + /** * dynamic_flags specifies chain state properties that can be * expressed in one bit. @@ -104,7 +108,6 @@ namespace graphene { namespace chain { }} FC_REFLECT_DERIVED( graphene::chain::dynamic_global_property_object, (graphene::db::object), - (random) (head_block_number) (head_block_id) (time) @@ -114,6 +117,7 @@ FC_REFLECT_DERIVED( graphene::chain::dynamic_global_property_object, (graphene:: (accounts_registered_this_interval) (recently_missed_count) (dynamic_flags) + (potential_witnesses) ) FC_REFLECT_DERIVED( graphene::chain::global_property_object, (graphene::db::object), diff --git a/libraries/chain/include/graphene/chain/protocol/block.hpp b/libraries/chain/include/graphene/chain/protocol/block.hpp index d9e882a1..34b0cfc2 100644 --- a/libraries/chain/include/graphene/chain/protocol/block.hpp +++ b/libraries/chain/include/graphene/chain/protocol/block.hpp @@ -27,8 +27,6 @@ namespace graphene { namespace chain { uint32_t block_num()const { return num_from_id(previous) + 1; } fc::time_point_sec timestamp; witness_id_type witness; - secret_hash_type next_secret_hash; - secret_hash_type previous_secret; checksum_type transaction_merkle_root; extensions_type extensions; @@ -53,7 +51,6 @@ namespace graphene { namespace chain { } } // graphene::chain -FC_REFLECT( graphene::chain::block_header, (previous)(timestamp)(witness) - (next_secret_hash)(previous_secret)(transaction_merkle_root)(extensions) ) +FC_REFLECT( graphene::chain::block_header, (previous)(timestamp)(witness)(transaction_merkle_root)(extensions) ) FC_REFLECT_DERIVED( graphene::chain::signed_block_header, (graphene::chain::block_header), (witness_signature) ) FC_REFLECT_DERIVED( graphene::chain::signed_block, (graphene::chain::signed_block_header), (transactions) ) diff --git a/libraries/chain/include/graphene/chain/witness_object.hpp b/libraries/chain/include/graphene/chain/witness_object.hpp index 17b94ed5..0b392eb8 100644 --- a/libraries/chain/include/graphene/chain/witness_object.hpp +++ b/libraries/chain/include/graphene/chain/witness_object.hpp @@ -32,6 +32,7 @@ namespace graphene { namespace chain { static const uint8_t type_id = witness_object_type; account_id_type witness_account; + uint32_t last_slot_num = 0; public_key_type signing_key; secret_hash_type next_secret_hash; secret_hash_type previous_secret; @@ -45,6 +46,7 @@ namespace graphene { namespace chain { struct by_account; struct by_vote_id; + struct by_last_block; using witness_multi_index_type = multi_index_container< witness_object, indexed_by< @@ -64,9 +66,8 @@ namespace graphene { namespace chain { FC_REFLECT_DERIVED( graphene::chain::witness_object, (graphene::db::object), (witness_account) + (last_slot_num) (signing_key) - (next_secret_hash) - (previous_secret) (pay_vb) (vote_id) (total_votes) diff --git a/libraries/wallet/wallet.cpp b/libraries/wallet/wallet.cpp index f34dbdb7..b61c752c 100644 --- a/libraries/wallet/wallet.cpp +++ b/libraries/wallet/wallet.cpp @@ -478,7 +478,6 @@ public: result["chain_id"] = chain_props.chain_id; result["active_witnesses"] = global_props.active_witnesses; result["active_committee_members"] = global_props.active_committee_members; - result["entropy"] = dynamic_props.random; return result; } chain_property_object get_chain_properties() const diff --git a/programs/size_checker/main.cpp b/programs/size_checker/main.cpp index 5950f6aa..0d3fc7f1 100644 --- a/programs/size_checker/main.cpp +++ b/programs/size_checker/main.cpp @@ -64,6 +64,20 @@ int main( int argc, char** argv ) { graphene::chain::operation op; + + vector witnesses; witnesses.resize(50); + for( uint32_t i = 0; i < 60*60*24*30; ++i ) + { + witnesses[ rand() % 50 ]++; + } + + std::sort( witnesses.begin(), witnesses.end() ); + idump((witnesses.back() - witnesses.front()) ); + idump((60*60*24*30/50)); + idump(("deviation: ")((60*60*24*30/50-witnesses.front())/(60*60*24*30/50.0))); + + idump( (witnesses) ); + for( int32_t i = 0; i < op.count(); ++i ) { op.set_which(i); @@ -85,6 +99,7 @@ int main( int argc, char** argv ) std::cout << "\n"; } std::cout << "]\n"; + std::cerr << "Size of block header: " << sizeof( block_header ) << " " << fc::raw::pack_size( block_header() ) << "\n"; } catch ( const fc::exception& e ){ edump((e.to_detail_string())); } idump((sizeof(signed_block))); From 30296d9c3603c1f58e0ba76bf72507407272bfa6 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Tue, 25 Aug 2015 14:46:56 -0400 Subject: [PATCH 2/8] database.hpp: Simplify get_scheduled_witness() return value --- libraries/chain/db_block.cpp | 4 +- libraries/chain/db_witness_schedule.cpp | 6 +- .../chain/include/graphene/chain/database.hpp | 6 +- libraries/plugins/witness/witness.cpp | 2 +- tests/app/main.cpp | 2 +- tests/benchmarks/genesis_allocation.cpp | 4 +- tests/common/database_fixture.cpp | 2 +- tests/intense/block_tests.cpp | 2 +- tests/tests/block_tests.cpp | 60 +++++++++---------- tests/tests/operation_tests2.cpp | 10 ++-- 10 files changed, 48 insertions(+), 50 deletions(-) diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp index 99131146..94b02511 100644 --- a/libraries/chain/db_block.cpp +++ b/libraries/chain/db_block.cpp @@ -257,7 +257,7 @@ signed_block database::_generate_block( uint32_t skip = get_node_properties().skip_flags; uint32_t slot_num = get_slot_at_time( when ); FC_ASSERT( slot_num > 0 ); - witness_id_type scheduled_witness = get_scheduled_witness( slot_num ).first; + witness_id_type scheduled_witness = get_scheduled_witness( slot_num ); FC_ASSERT( scheduled_witness == witness_id ); const auto& witness_obj = witness_id(*this); @@ -566,7 +566,7 @@ const witness_object& database::validate_block_header( uint32_t skip, const sign uint32_t slot_num = get_slot_at_time( next_block.timestamp ); FC_ASSERT( slot_num > 0 ); - witness_id_type scheduled_witness = get_scheduled_witness( slot_num ).first; + witness_id_type scheduled_witness = get_scheduled_witness( slot_num ); FC_ASSERT( next_block.witness == scheduled_witness ); } diff --git a/libraries/chain/db_witness_schedule.cpp b/libraries/chain/db_witness_schedule.cpp index 2cb8b711..90aab09e 100644 --- a/libraries/chain/db_witness_schedule.cpp +++ b/libraries/chain/db_witness_schedule.cpp @@ -23,10 +23,10 @@ namespace graphene { namespace chain { -pair database::get_scheduled_witness(uint32_t slot_num)const +witness_id_type database::get_scheduled_witness(uint32_t slot_num)const { if( slot_num == 0 ) - return pair(witness_id_type(), false); + return witness_id_type(); const witness_schedule_object& wso = witness_schedule_id_type()(*this); @@ -53,7 +53,7 @@ pair database::get_scheduled_witness(uint32_t slot_num)co assert( false ); } } - return pair(wid, slot_is_near); + return wid; } fc::time_point_sec database::get_slot_time(uint32_t slot_num)const diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp index e1a56e90..99df3cb0 100644 --- a/libraries/chain/include/graphene/chain/database.hpp +++ b/libraries/chain/include/graphene/chain/database.hpp @@ -220,11 +220,9 @@ namespace graphene { namespace chain { * Use the get_slot_time() and get_slot_at_time() functions * to convert between slot_num and timestamp. * - * Passing slot_num == 0 returns (witness_id_type(), false) - * - * The bool value is true if near schedule, false if far schedule. + * Passing slot_num == 0 returns witness_id_type() */ - pair get_scheduled_witness(uint32_t slot_num)const; + witness_id_type get_scheduled_witness(uint32_t slot_num)const; /** * Get the time at which the given slot occurs. diff --git a/libraries/plugins/witness/witness.cpp b/libraries/plugins/witness/witness.cpp index 637447ee..9290837d 100644 --- a/libraries/plugins/witness/witness.cpp +++ b/libraries/plugins/witness/witness.cpp @@ -177,7 +177,7 @@ void witness_plugin::block_production_loop() // is anyone scheduled to produce now or one second in the future? const fc::time_point_sec now = graphene::time::now(); uint32_t slot = db.get_slot_at_time( now ); - graphene::chain::witness_id_type scheduled_witness = db.get_scheduled_witness( slot ).first; + graphene::chain::witness_id_type scheduled_witness = db.get_scheduled_witness( slot ); fc::time_point_sec scheduled_time = db.get_slot_time( slot ); graphene::chain::public_key_type scheduled_key = scheduled_witness( db ).signing_key; diff --git a/tests/app/main.cpp b/tests/app/main.cpp index 1e7bfcad..b498ec01 100644 --- a/tests/app/main.cpp +++ b/tests/app/main.cpp @@ -128,7 +128,7 @@ BOOST_AUTO_TEST_CASE( two_node_network ) auto block_1 = db2->generate_block( db2->get_slot_time(1), - db2->get_scheduled_witness(1).first, + db2->get_scheduled_witness(1), committee_key, database::skip_nothing); diff --git a/tests/benchmarks/genesis_allocation.cpp b/tests/benchmarks/genesis_allocation.cpp index eb93700a..5dedaedf 100644 --- a/tests/benchmarks/genesis_allocation.cpp +++ b/tests/benchmarks/genesis_allocation.cpp @@ -86,7 +86,7 @@ BOOST_AUTO_TEST_CASE( genesis_and_persistence_bench ) int blocks_out = 0; auto witness_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); auto aw = db.get_global_properties().active_witnesses; - auto b = db.generate_block( db.get_slot_time( 1 ), db.get_scheduled_witness( 1 ).first, witness_priv_key, ~0 ); + auto b = db.generate_block( db.get_slot_time( 1 ), db.get_scheduled_witness( 1 ), witness_priv_key, ~0 ); start_time = fc::time_point::now(); /* TODO: get this buliding again @@ -97,7 +97,7 @@ BOOST_AUTO_TEST_CASE( genesis_and_persistence_bench ) db.push_transaction(trx, ~0); aw = db.get_global_properties().active_witnesses; - b = db.generate_block( db.get_slot_time( 1 ), db.get_scheduled_witness( 1 ).first, witness_priv_key, ~0 ); + b = db.generate_block( db.get_slot_time( 1 ), db.get_scheduled_witness( 1 ), witness_priv_key, ~0 ); } */ ilog("Pushed ${c} blocks (1 op each, no validation) in ${t} milliseconds.", diff --git a/tests/common/database_fixture.cpp b/tests/common/database_fixture.cpp index 53cbd00a..61fb036d 100644 --- a/tests/common/database_fixture.cpp +++ b/tests/common/database_fixture.cpp @@ -297,7 +297,7 @@ signed_block database_fixture::generate_block(uint32_t skip, const fc::ecc::priv skip |= database::skip_undo_history_check; // skip == ~0 will skip checks specified in database::validation_steps return db.generate_block(db.get_slot_time(miss_blocks + 1), - db.get_scheduled_witness(miss_blocks + 1).first, + db.get_scheduled_witness(miss_blocks + 1), key, skip); } diff --git a/tests/intense/block_tests.cpp b/tests/intense/block_tests.cpp index e9f1196e..fd77132e 100644 --- a/tests/intense/block_tests.cpp +++ b/tests/intense/block_tests.cpp @@ -279,7 +279,7 @@ BOOST_FIXTURE_TEST_CASE( witness_order_mc_test, database_fixture ) { wdump( (db.head_block_num()) ); } - witness_id_type wid = db.get_scheduled_witness( 1 ).first; + witness_id_type wid = db.get_scheduled_witness( 1 ); full_schedule.push_back( wid ); cur_round.push_back( wid ); if( cur_round.size() == num_witnesses ) diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp index 427fdd51..d8301856 100644 --- a/tests/tests/block_tests.cpp +++ b/tests/tests/block_tests.cpp @@ -131,13 +131,13 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) { database db; db.open(data_dir.path(), make_genesis ); - b = db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); + b = db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); for( uint32_t i = 1; i < 200; ++i ) { BOOST_CHECK( db.head_block_id() == b.id() ); witness_id_type prev_witness = b.witness; - witness_id_type cur_witness = db.get_scheduled_witness(1).first; + witness_id_type cur_witness = db.get_scheduled_witness(1); BOOST_CHECK( cur_witness != prev_witness ); b = db.generate_block(db.get_slot_time(1), cur_witness, init_account_priv_key, database::skip_nothing); BOOST_CHECK( b.witness == cur_witness ); @@ -152,7 +152,7 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) { BOOST_CHECK( db.head_block_id() == b.id() ); witness_id_type prev_witness = b.witness; - witness_id_type cur_witness = db.get_scheduled_witness(1).first; + witness_id_type cur_witness = db.get_scheduled_witness(1); BOOST_CHECK( cur_witness != prev_witness ); b = db.generate_block(db.get_slot_time(1), cur_witness, init_account_priv_key, database::skip_nothing); } @@ -179,7 +179,7 @@ BOOST_AUTO_TEST_CASE( undo_block ) { now = db.get_slot_time(1); time_stack.push_back( now ); - auto b = db.generate_block( now, db.get_scheduled_witness( 1 ).first, init_account_priv_key, database::skip_nothing ); + auto b = db.generate_block( now, db.get_scheduled_witness( 1 ), init_account_priv_key, database::skip_nothing ); } BOOST_CHECK( db.head_block_num() == 5 ); BOOST_CHECK( db.head_block_time() == now ); @@ -202,7 +202,7 @@ BOOST_AUTO_TEST_CASE( undo_block ) { now = db.get_slot_time(1); time_stack.push_back( now ); - auto b = db.generate_block( now, db.get_scheduled_witness( 1 ).first, init_account_priv_key, database::skip_nothing ); + auto b = db.generate_block( now, db.get_scheduled_witness( 1 ), init_account_priv_key, database::skip_nothing ); } BOOST_CHECK( db.head_block_num() == 7 ); } @@ -227,20 +227,20 @@ BOOST_AUTO_TEST_CASE( fork_blocks ) auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); for( uint32_t i = 0; i < 10; ++i ) { - auto b = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); + auto b = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); try { PUSH_BLOCK( db2, b ); } FC_CAPTURE_AND_RETHROW( ("db2") ); } for( uint32_t i = 10; i < 13; ++i ) { - auto b = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); + auto b = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); } string db1_tip = db1.head_block_id().str(); uint32_t next_slot = 3; for( uint32_t i = 13; i < 16; ++i ) { - auto b = db2.generate_block(db2.get_slot_time(next_slot), db2.get_scheduled_witness(next_slot).first, init_account_priv_key, database::skip_nothing); + auto b = db2.generate_block(db2.get_slot_time(next_slot), db2.get_scheduled_witness(next_slot), init_account_priv_key, database::skip_nothing); next_slot = 1; // notify both databases of the new block. // only db2 should switch to the new fork, db1 should not @@ -255,7 +255,7 @@ BOOST_AUTO_TEST_CASE( fork_blocks ) BOOST_CHECK_EQUAL(db1.head_block_num(), 13); BOOST_CHECK_EQUAL(db2.head_block_num(), 13); { - auto b = db2.generate_block(db2.get_slot_time(1), db2.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); + auto b = db2.generate_block(db2.get_slot_time(1), db2.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); good_block = b; b.transactions.emplace_back(signed_transaction()); b.transactions.back().operations.emplace_back(transfer_operation()); @@ -289,18 +289,18 @@ BOOST_AUTO_TEST_CASE( out_of_order_blocks ) BOOST_CHECK( db1.get_chain_id() == db2.get_chain_id() ); auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); - auto b1 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); - auto b2 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); - auto b3 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); - auto b4 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); - auto b5 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); - auto b6 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); - auto b7 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); - auto b8 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); - auto b9 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); - auto b10 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); - auto b11 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); - auto b12 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); + auto b1 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + auto b2 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + auto b3 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + auto b4 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + auto b5 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + auto b6 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + auto b7 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + auto b8 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + auto b9 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + auto b10 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + auto b11 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + auto b12 = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); BOOST_CHECK_EQUAL(db1.head_block_num(), 12); BOOST_CHECK_EQUAL(db2.head_block_num(), 0); PUSH_BLOCK( db2, b1 ); @@ -351,7 +351,7 @@ BOOST_AUTO_TEST_CASE( undo_pending ) trx.operations.push_back(t); PUSH_TX( db, trx, ~0 ); - auto b = db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1).first, init_account_priv_key, ~0); + auto b = db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1), init_account_priv_key, ~0); } signed_transaction trx; @@ -366,7 +366,7 @@ BOOST_AUTO_TEST_CASE( undo_pending ) //sign( trx, init_account_priv_key ); PUSH_TX( db, trx ); - auto b = db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); + auto b = db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); BOOST_CHECK(nathan_id(db).name == "nathan"); @@ -424,14 +424,14 @@ BOOST_AUTO_TEST_CASE( switch_forks_undo_create ) // db2 : B C D auto aw = db1.get_global_properties().active_witnesses; - auto b = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); + auto b = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); BOOST_CHECK(nathan_id(db1).name == "nathan"); - b = db2.generate_block(db2.get_slot_time(1), db2.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); + b = db2.generate_block(db2.get_slot_time(1), db2.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); db1.push_block(b); aw = db2.get_global_properties().active_witnesses; - b = db2.generate_block(db2.get_slot_time(1), db2.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); + b = db2.generate_block(db2.get_slot_time(1), db2.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); db1.push_block(b); GRAPHENE_CHECK_THROW(nathan_id(db1), fc::exception); @@ -439,7 +439,7 @@ BOOST_AUTO_TEST_CASE( switch_forks_undo_create ) PUSH_TX( db2, trx ); aw = db2.get_global_properties().active_witnesses; - b = db2.generate_block(db2.get_slot_time(1), db2.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); + b = db2.generate_block(db2.get_slot_time(1), db2.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); db1.push_block(b); BOOST_CHECK(nathan_id(db1).name == "nathan"); @@ -489,7 +489,7 @@ BOOST_AUTO_TEST_CASE( duplicate_transactions ) GRAPHENE_CHECK_THROW(PUSH_TX( db1, trx, skip_sigs ), fc::exception); - auto b = db1.generate_block( db1.get_slot_time(1), db1.get_scheduled_witness( 1 ).first, init_account_priv_key, skip_sigs ); + auto b = db1.generate_block( db1.get_slot_time(1), db1.get_scheduled_witness( 1 ), init_account_priv_key, skip_sigs ); PUSH_BLOCK( db2, b, skip_sigs ); GRAPHENE_CHECK_THROW(PUSH_TX( db1, trx, skip_sigs ), fc::exception); @@ -515,7 +515,7 @@ BOOST_AUTO_TEST_CASE( tapos ) public_key_type init_account_pub_key = init_account_priv_key.get_public_key(); const graphene::db::index& account_idx = db1.get_index(protocol_ids, account_object_type); - auto b = db1.generate_block( db1.get_slot_time(1), db1.get_scheduled_witness( 1 ).first, init_account_priv_key, database::skip_nothing); + auto b = db1.generate_block( db1.get_slot_time(1), db1.get_scheduled_witness( 1 ), init_account_priv_key, database::skip_nothing); signed_transaction trx; //This transaction must be in the next block after its reference, or it is invalid. @@ -531,7 +531,7 @@ BOOST_AUTO_TEST_CASE( tapos ) trx.operations.push_back(cop); trx.sign( init_account_priv_key, db1.get_chain_id() ); db1.push_transaction(trx); - b = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1).first, init_account_priv_key, database::skip_nothing); + b = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); trx.clear(); transfer_operation t; diff --git a/tests/tests/operation_tests2.cpp b/tests/tests/operation_tests2.cpp index 28192b2e..59c34c92 100644 --- a/tests/tests/operation_tests2.cpp +++ b/tests/tests/operation_tests2.cpp @@ -1098,7 +1098,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test ) BOOST_CHECK(db.find_object(balance_id_type(1)) != nullptr); auto slot = db.get_slot_at_time(starting_time); - db.generate_block(starting_time, db.get_scheduled_witness(slot).first, init_account_priv_key, skip_flags); + db.generate_block(starting_time, db.get_scheduled_witness(slot), init_account_priv_key, skip_flags); set_expiration( db, trx ); const balance_object& vesting_balance_1 = balance_id_type(2)(db); @@ -1149,9 +1149,9 @@ BOOST_AUTO_TEST_CASE( balance_object_test ) // Attempting to claim twice within a day GRAPHENE_CHECK_THROW(db.push_transaction(trx), balance_claim_claimed_too_often); - db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1).first, init_account_priv_key, skip_flags); + db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1), init_account_priv_key, skip_flags); slot = db.get_slot_at_time(vesting_balance_1.vesting_policy->begin_timestamp + 60); - db.generate_block(db.get_slot_time(slot), db.get_scheduled_witness(slot).first, init_account_priv_key, skip_flags); + db.generate_block(db.get_slot_time(slot), db.get_scheduled_witness(slot), init_account_priv_key, skip_flags); set_expiration( db, trx ); op.balance_to_claim = vesting_balance_1.id; @@ -1175,9 +1175,9 @@ BOOST_AUTO_TEST_CASE( balance_object_test ) // Attempting to claim twice within a day GRAPHENE_CHECK_THROW(db.push_transaction(trx), balance_claim_claimed_too_often); - db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1).first, init_account_priv_key, skip_flags); + db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1), init_account_priv_key, skip_flags); slot = db.get_slot_at_time(db.head_block_time() + fc::days(1)); - db.generate_block(db.get_slot_time(slot), db.get_scheduled_witness(slot).first, init_account_priv_key, skip_flags); + db.generate_block(db.get_slot_time(slot), db.get_scheduled_witness(slot), init_account_priv_key, skip_flags); set_expiration( db, trx ); op.total_claimed = vesting_balance_2.balance; From c2e5432a3093c338e2e8cdd6df3bb3bfabf206bf Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Tue, 25 Aug 2015 17:54:04 -0400 Subject: [PATCH 3/8] Remove block randomness and rewrite witness scheduling --- libraries/app/api.cpp | 1 - libraries/chain/db_block.cpp | 21 - libraries/chain/db_init.cpp | 31 +- libraries/chain/db_maint.cpp | 8 +- libraries/chain/db_management.cpp | 1 + libraries/chain/db_update.cpp | 27 +- libraries/chain/db_witness_schedule.cpp | 175 +++----- .../chain/include/graphene/chain/config.hpp | 3 + .../chain/include/graphene/chain/database.hpp | 8 - .../graphene/chain/global_property_object.hpp | 27 +- .../include/graphene/chain/protocol/block.hpp | 5 +- .../include/graphene/chain/protocol/types.hpp | 5 - .../include/graphene/chain/witness_object.hpp | 4 +- .../chain/witness_schedule_object.hpp | 88 ---- .../graphene/chain/witness_scheduler.hpp | 417 ------------------ .../graphene/chain/witness_scheduler_rng.hpp | 124 ------ libraries/wallet/wallet.cpp | 1 - programs/size_checker/main.cpp | 1 + tests/intense/block_tests.cpp | 1 - tests/tests/basic_tests.cpp | 130 ------ tests/tests/block_tests.cpp | 30 +- tests/tests/operation_tests2.cpp | 37 +- 22 files changed, 123 insertions(+), 1022 deletions(-) delete mode 100644 libraries/chain/include/graphene/chain/witness_schedule_object.hpp delete mode 100644 libraries/chain/include/graphene/chain/witness_scheduler.hpp delete mode 100644 libraries/chain/include/graphene/chain/witness_scheduler_rng.hpp diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index ef7f3a13..a946491d 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -781,7 +781,6 @@ namespace graphene { namespace app { break; } case impl_block_summary_object_type:{ } case impl_account_transaction_history_object_type:{ - } case impl_witness_schedule_object_type: { } case impl_chain_property_object_type: { } } diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp index 94b02511..a78fb506 100644 --- a/libraries/chain/db_block.cpp +++ b/libraries/chain/db_block.cpp @@ -267,24 +267,6 @@ signed_block database::_generate_block( _pending_block.timestamp = when; - // Genesis witnesses start with a default initial secret - if( witness_obj.next_secret_hash == secret_hash_type::hash( secret_hash_type() ) ) - { - _pending_block.previous_secret = secret_hash_type(); - } - else - { - secret_hash_type::encoder last_enc; - fc::raw::pack( last_enc, block_signing_private_key ); - fc::raw::pack( last_enc, witness_obj.previous_secret ); - _pending_block.previous_secret = last_enc.result(); - } - - secret_hash_type::encoder next_enc; - fc::raw::pack( next_enc, block_signing_private_key ); - fc::raw::pack( next_enc, _pending_block.previous_secret ); - _pending_block.next_secret_hash = secret_hash_type::hash(next_enc.result()); - _pending_block.transaction_merkle_root = _pending_block.calculate_merkle_root(); _pending_block.witness = witness_id; @@ -404,7 +386,6 @@ void database::_apply_block( const signed_block& next_block ) ++_current_trx_in_block; } - update_witness_schedule(next_block); update_global_dynamic_data(next_block); update_signing_witness(signing_witness, next_block); @@ -555,8 +536,6 @@ const witness_object& database::validate_block_header( uint32_t skip, const sign FC_ASSERT( _pending_block.previous == next_block.previous, "", ("pending.prev",_pending_block.previous)("next.prev",next_block.previous) ); FC_ASSERT( _pending_block.timestamp <= next_block.timestamp, "", ("_pending_block.timestamp",_pending_block.timestamp)("next",next_block.timestamp)("blocknum",next_block.block_num()) ); const witness_object& witness = next_block.witness(*this); - FC_ASSERT( secret_hash_type::hash( next_block.previous_secret ) == witness.next_secret_hash, "", - ("previous_secret", next_block.previous_secret)("next_secret_hash", witness.next_secret_hash)); if( !(skip&skip_witness_signature) ) FC_ASSERT( next_block.validate_signee( witness.signing_key ) ); diff --git a/libraries/chain/db_init.cpp b/libraries/chain/db_init.cpp index 3b7a07f3..3376b2b8 100644 --- a/libraries/chain/db_init.cpp +++ b/libraries/chain/db_init.cpp @@ -30,7 +30,6 @@ #include #include #include -#include #include #include @@ -107,9 +106,6 @@ const uint8_t withdraw_permission_object::type_id; const uint8_t witness_object::space_id; const uint8_t witness_object::type_id; -const uint8_t witness_schedule_object::space_id; -const uint8_t witness_schedule_object::type_id; - const uint8_t worker_object::space_id; const uint8_t worker_object::type_id; @@ -193,7 +189,6 @@ void database::initialize_indexes() add_index< primary_index> >(); add_index< primary_index> >(); add_index< primary_index> >(); - add_index< primary_index> >(); add_index< primary_index > >(); } @@ -317,6 +312,7 @@ void database::init_genesis(const genesis_state_type& genesis_state) p.time = genesis_state.initial_timestamp; p.dynamic_flags = 0; p.witness_budget = 0; + p.recent_slots_filled = fc::uint128::max_value(); }); FC_ASSERT( (genesis_state.immutable_parameters.min_witness_count & 1) == 1, "min_witness_count must be odd" ); @@ -538,31 +534,6 @@ void database::init_genesis(const genesis_state_type& genesis_state) } }); - // Initialize witness schedule -#ifndef NDEBUG - const witness_schedule_object& wso = -#endif - create([&](witness_schedule_object& _wso) - { - memset(_wso.rng_seed.begin(), 0, _wso.rng_seed.size()); - - witness_scheduler_rng rng(_wso.rng_seed.begin(), GRAPHENE_NEAR_SCHEDULE_CTR_IV); - - auto init_witnesses = get_global_properties().active_witnesses; - - _wso.scheduler = witness_scheduler(); - _wso.scheduler._min_token_count = std::max(int(init_witnesses.size()) / 2, 1); - _wso.scheduler.update(init_witnesses); - - for( size_t i=0; i + #include #include @@ -27,7 +29,6 @@ #include #include #include -#include #include namespace graphene { namespace chain { @@ -206,11 +207,6 @@ void database::update_active_witnesses() }); }); - const witness_schedule_object& wso = witness_schedule_id_type()(*this); - modify(wso, [&](witness_schedule_object& _wso) - { - _wso.scheduler.update(gpo.active_witnesses); - }); } FC_CAPTURE_AND_RETHROW() } void database::update_active_committee_members() diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp index 52bc39b0..0a6c75ce 100644 --- a/libraries/chain/db_management.cpp +++ b/libraries/chain/db_management.cpp @@ -22,6 +22,7 @@ #include #include +#include namespace graphene { namespace chain { diff --git a/libraries/chain/db_update.cpp b/libraries/chain/db_update.cpp index ded6a04a..e4f8ab86 100644 --- a/libraries/chain/db_update.cpp +++ b/libraries/chain/db_update.cpp @@ -36,25 +36,18 @@ void database::update_global_dynamic_data( const signed_block& b ) const dynamic_global_property_object& _dgp = dynamic_global_property_id_type(0)(*this); - const auto& global_props = get_global_properties(); - auto delta_time = b.timestamp - _dgp.time; - auto missed_blocks = (delta_time.to_seconds() / global_props.parameters.block_interval) - 1; - if( _dgp.head_block_number == 0 ) - missed_blocks = 0; + uint32_t missed_blocks = get_slot_at_time( b.timestamp ); + assert( missed_blocks != 0 ); + missed_blocks--; // dynamic global properties updating modify( _dgp, [&]( dynamic_global_property_object& dgp ){ - secret_hash_type::encoder enc; - fc::raw::pack( enc, dgp.random ); - fc::raw::pack( enc, b.previous_secret ); - dgp.random = enc.result(); - if( _checkpoints.size() && _checkpoints.rbegin()->first >= b.block_num() ) dgp.recently_missed_count = 0; else if( missed_blocks ) - dgp.recently_missed_count += 4*missed_blocks; - else if( dgp.recently_missed_count > 4 ) - dgp.recently_missed_count -= 3; + dgp.recently_missed_count += GRAPHENE_RECENTLY_MISSED_COUNT_INCREMENT*missed_blocks; + else if( dgp.recently_missed_count > GRAPHENE_RECENTLY_MISSED_COUNT_INCREMENT ) + dgp.recently_missed_count -= GRAPHENE_RECENTLY_MISSED_COUNT_DECREMENT; else if( dgp.recently_missed_count > 0 ) dgp.recently_missed_count--; @@ -62,6 +55,10 @@ void database::update_global_dynamic_data( const signed_block& b ) dgp.head_block_id = b.id(); dgp.time = b.timestamp; dgp.current_witness = b.witness; + dgp.recent_slots_filled = ( + (dgp.recent_slots_filled << 1) + + 1) << missed_blocks; + dgp.current_aslot += missed_blocks+1; }); if( !(get_node_properties().skip_flags & skip_undo_history_check) ) @@ -80,6 +77,7 @@ void database::update_signing_witness(const witness_object& signing_witness, con { const global_property_object& gpo = get_global_properties(); const dynamic_global_property_object& dpo = get_dynamic_global_properties(); + uint64_t new_block_aslot = dpo.current_aslot + get_slot_at_time( new_block.timestamp ); share_type witness_pay = std::min( gpo.parameters.witness_pay_per_block, dpo.witness_budget ); @@ -92,8 +90,7 @@ void database::update_signing_witness(const witness_object& signing_witness, con modify( signing_witness, [&]( witness_object& _wit ) { - _wit.previous_secret = new_block.previous_secret; - _wit.next_secret_hash = new_block.next_secret_hash; + _wit.last_aslot = new_block_aslot; } ); } diff --git a/libraries/chain/db_witness_schedule.cpp b/libraries/chain/db_witness_schedule.cpp index 90aab09e..d12bf69e 100644 --- a/libraries/chain/db_witness_schedule.cpp +++ b/libraries/chain/db_witness_schedule.cpp @@ -15,45 +15,87 @@ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#pragma once #include - #include -#include +#include namespace graphene { namespace chain { -witness_id_type database::get_scheduled_witness(uint32_t slot_num)const +using boost::container::flat_set; + +witness_id_type database::get_scheduled_witness( uint32_t slot_num )const { - if( slot_num == 0 ) - return witness_id_type(); + // + // Each witness gets an arbitration key H(time, witness_id). + // The witness with the smallest key is selected to go first. + // + // As opposed to just using H(time) to determine an index into + // an array of eligible witnesses, this has the following desirable + // properties: + // + // - Avoid dynamic memory allocation + // - Decreases (but does not eliminate) the probability that a + // missed block will change the witness assigned to a future slot + // + // The hash function is xorshift* as given in + // [1] https://en.wikipedia.org/wiki/Xorshift#Xorshift.2A + // - const witness_schedule_object& wso = witness_schedule_id_type()(*this); + const flat_set< witness_id_type >& active_witnesses = get_global_properties().active_witnesses; + uint32_t n = active_witnesses.size(); + uint64_t min_witness_separation = (n / 2)+1; + uint64_t current_aslot = get_dynamic_global_properties().current_aslot + slot_num; - // ask the near scheduler who goes in the given slot - witness_id_type wid; - bool slot_is_near = wso.scheduler.get_slot(slot_num-1, wid); - if( ! slot_is_near ) + uint64_t start_of_current_round_aslot = current_aslot - (current_aslot % n); + uint64_t first_ineligible_aslot = std::min( + start_of_current_round_aslot, current_aslot - min_witness_separation ); + // + // overflow analysis of above subtraction: + // + // we always have min_witness_separation <= n, so + // if current_aslot < min_witness_separation it follows that + // start_of_current_round_aslot == 0 + // + // therefore result of above min() is 0 when subtraction overflows + // + + first_ineligible_aslot = std::max( first_ineligible_aslot, uint64_t( 1 ) ); + + uint64_t best_k = 0; + witness_id_type best_wit; + bool success = false; + + uint64_t now_hi = get_slot_time( slot_num ).sec_since_epoch(); + now_hi <<= 32; + + for( const witness_id_type& wit_id : active_witnesses ) { - // if the near scheduler doesn't know, we have to extend it to - // a far scheduler. - // n.b. instantiating it is slow, but block gaps long enough to - // need it are likely pretty rare. + const witness_object& wit = wit_id(*this); + if( wit.last_aslot >= first_ineligible_aslot ) + continue; - witness_scheduler_rng far_rng(wso.rng_seed.begin(), GRAPHENE_FAR_SCHEDULE_CTR_IV); - - far_future_witness_scheduler far_scheduler = - far_future_witness_scheduler(wso.scheduler, far_rng); - if( !far_scheduler.get_slot(slot_num-1, wid) ) + uint64_t k = now_hi + uint64_t(wit_id); + k ^= (k >> 12); + k ^= (k << 25); + k ^= (k >> 27); + k *= 2685821657736338717ULL; + if( k >= best_k ) { - // no scheduled witness -- somebody set up us the bomb - // n.b. this code path is impossible, the present - // implementation of far_future_witness_scheduler - // returns true unconditionally - assert( false ); + best_k = k; + best_wit = wit_id; + success = true; } } - return wid; + + // the above loop should choose at least 1 because + // at most K elements are susceptible to the filter, + // otherwise we have an inconsistent database (such as + // wit.last_aslot values that are non-unique or in the future) + + assert( success ); + return best_wit; } fc::time_point_sec database::get_slot_time(uint32_t slot_num)const @@ -98,89 +140,10 @@ uint32_t database::get_slot_at_time(fc::time_point_sec when)const return (when - first_slot_time).to_seconds() / block_interval() + 1; } -vector database::get_near_witness_schedule()const -{ - const witness_schedule_object& wso = witness_schedule_id_type()(*this); - - vector result; - result.reserve(wso.scheduler.size()); - uint32_t slot_num = 0; - witness_id_type wid; - - while( wso.scheduler.get_slot(slot_num++, wid) ) - result.emplace_back(wid); - - return result; -} - -void database::update_witness_schedule(const signed_block& next_block) -{ - auto start = fc::time_point::now(); - const global_property_object& gpo = get_global_properties(); - const witness_schedule_object& wso = get(witness_schedule_id_type()); - uint32_t schedule_needs_filled = gpo.active_witnesses.size(); - uint32_t schedule_slot = get_slot_at_time(next_block.timestamp); - - // We shouldn't be able to generate _pending_block with timestamp - // in the past, and incoming blocks from the network with timestamp - // in the past shouldn't be able to make it this far without - // triggering FC_ASSERT elsewhere - - assert( schedule_slot > 0 ); - witness_id_type first_witness; - bool slot_is_near = wso.scheduler.get_slot( schedule_slot-1, first_witness ); - - witness_id_type wit; - - const dynamic_global_property_object& dpo = get_dynamic_global_properties(); - - assert( dpo.random.data_size() == witness_scheduler_rng::seed_length ); - assert( witness_scheduler_rng::seed_length == wso.rng_seed.size() ); - - modify(wso, [&](witness_schedule_object& _wso) - { - _wso.slots_since_genesis += schedule_slot; - witness_scheduler_rng rng(wso.rng_seed.data, _wso.slots_since_genesis); - - _wso.scheduler._min_token_count = std::max(int(gpo.active_witnesses.size()) / 2, 1); - - if( slot_is_near ) - { - uint32_t drain = schedule_slot; - while( drain > 0 ) - { - if( _wso.scheduler.size() == 0 ) - break; - _wso.scheduler.consume_schedule(); - --drain; - } - } - else - { - _wso.scheduler.reset_schedule( first_witness ); - } - while( !_wso.scheduler.get_slot(schedule_needs_filled, wit) ) - { - if( _wso.scheduler.produce_schedule(rng) & emit_turn ) - memcpy(_wso.rng_seed.begin(), dpo.random.data(), dpo.random.data_size()); - } - _wso.last_scheduling_block = next_block.block_num(); - _wso.recent_slots_filled = ( - (_wso.recent_slots_filled << 1) - + 1) << (schedule_slot - 1); - }); - auto end = fc::time_point::now(); - static uint64_t total_time = 0; - static uint64_t calls = 0; - total_time += (end - start).count(); - if( ++calls % 1000 == 0 ) - idump( ( double(total_time/1000000.0)/calls) ); -} - uint32_t database::witness_participation_rate()const { - const witness_schedule_object& wso = get(witness_schedule_id_type()); - return uint64_t(GRAPHENE_100_PERCENT) * wso.recent_slots_filled.popcount() / 128; + const dynamic_global_property_object& dpo = get_dynamic_global_properties(); + return uint64_t(GRAPHENE_100_PERCENT) * dpo.recent_slots_filled.popcount() / 128; } } } diff --git a/libraries/chain/include/graphene/chain/config.hpp b/libraries/chain/include/graphene/chain/config.hpp index f77f7144..38d3db6d 100644 --- a/libraries/chain/include/graphene/chain/config.hpp +++ b/libraries/chain/include/graphene/chain/config.hpp @@ -139,6 +139,9 @@ #define GRAPHENE_MAX_INTEREST_APR uint16_t( 10000 ) +#define GRAPHENE_RECENTLY_MISSED_COUNT_INCREMENT 4 +#define GRAPHENE_RECENTLY_MISSED_COUNT_DECREMENT 3 + /** * Reserved Account IDs with special meaning */ diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp index 99df3cb0..ec197889 100644 --- a/libraries/chain/include/graphene/chain/database.hpp +++ b/libraries/chain/include/graphene/chain/database.hpp @@ -244,11 +244,6 @@ namespace graphene { namespace chain { */ uint32_t get_slot_at_time(fc::time_point_sec when)const; - /** - * Get the near schedule. - */ - vector get_near_witness_schedule()const; - //////////////////// db_getter.cpp //////////////////// const chain_id_type& get_chain_id()const; @@ -451,9 +446,6 @@ namespace graphene { namespace chain { void update_maintenance_flag( bool new_maintenance_flag ); void update_withdraw_permissions(); - //////////////////// db_witness_schedule.cpp //////////////////// - void update_witness_schedule(const signed_block& next_block); /// no-op except for scheduling blocks - ///Steps performed only at maintenance intervals ///@{ diff --git a/libraries/chain/include/graphene/chain/global_property_object.hpp b/libraries/chain/include/graphene/chain/global_property_object.hpp index dacb5ce2..c52997fb 100644 --- a/libraries/chain/include/graphene/chain/global_property_object.hpp +++ b/libraries/chain/include/graphene/chain/global_property_object.hpp @@ -42,7 +42,7 @@ namespace graphene { namespace chain { chain_parameters parameters; optional pending_parameters; - uint32_t next_available_vote_id = 0; + uint32_t next_available_vote_id = 0; vector active_committee_members; // updated once per maintenance interval flat_set active_witnesses; // updated once per maintenance interval // n.b. witness scheduling is done by witness_schedule object @@ -64,7 +64,6 @@ namespace graphene { namespace chain { static const uint8_t space_id = implementation_ids; static const uint8_t type_id = impl_dynamic_global_property_object_type; - secret_hash_type random; uint32_t head_block_number = 0; block_id_type head_block_id; time_point_sec time; @@ -74,13 +73,28 @@ namespace graphene { namespace chain { share_type witness_budget; uint32_t accounts_registered_this_interval = 0; /** - * Every time a block is missed this increases by 2, every time a block is found it decreases by 1 it is - * never less than 0 + * Every time a block is missed this increases by + * RECENTLY_MISSED_COUNT_INCREMENT, + * every time a block is found it decreases by + * RECENTLY_MISSED_COUNT_DECREMENT. It is + * never less than 0. * - * If the recently_missed_count hits 2*UNDO_HISTORY then no ew blocks may be pushed. + * If the recently_missed_count hits 2*UNDO_HISTORY then no new blocks may be pushed. */ uint32_t recently_missed_count = 0; + /** + * The current absolute slot number. Equal to the total + * number of slots since genesis. Also equal to the total + * number of missed slots plus head_block_number. + */ + uint64_t current_aslot = 0; + + /** + * used to compute witness participation. + */ + fc::uint128_t recent_slots_filled; + /** * dynamic_flags specifies chain state properties that can be * expressed in one bit. @@ -104,7 +118,6 @@ namespace graphene { namespace chain { }} FC_REFLECT_DERIVED( graphene::chain::dynamic_global_property_object, (graphene::db::object), - (random) (head_block_number) (head_block_id) (time) @@ -113,6 +126,8 @@ FC_REFLECT_DERIVED( graphene::chain::dynamic_global_property_object, (graphene:: (witness_budget) (accounts_registered_this_interval) (recently_missed_count) + (current_aslot) + (recent_slots_filled) (dynamic_flags) ) diff --git a/libraries/chain/include/graphene/chain/protocol/block.hpp b/libraries/chain/include/graphene/chain/protocol/block.hpp index d9e882a1..34b0cfc2 100644 --- a/libraries/chain/include/graphene/chain/protocol/block.hpp +++ b/libraries/chain/include/graphene/chain/protocol/block.hpp @@ -27,8 +27,6 @@ namespace graphene { namespace chain { uint32_t block_num()const { return num_from_id(previous) + 1; } fc::time_point_sec timestamp; witness_id_type witness; - secret_hash_type next_secret_hash; - secret_hash_type previous_secret; checksum_type transaction_merkle_root; extensions_type extensions; @@ -53,7 +51,6 @@ namespace graphene { namespace chain { } } // graphene::chain -FC_REFLECT( graphene::chain::block_header, (previous)(timestamp)(witness) - (next_secret_hash)(previous_secret)(transaction_merkle_root)(extensions) ) +FC_REFLECT( graphene::chain::block_header, (previous)(timestamp)(witness)(transaction_merkle_root)(extensions) ) FC_REFLECT_DERIVED( graphene::chain::signed_block_header, (graphene::chain::block_header), (witness_signature) ) FC_REFLECT_DERIVED( graphene::chain::signed_block, (graphene::chain::signed_block_header), (transactions) ) diff --git a/libraries/chain/include/graphene/chain/protocol/types.hpp b/libraries/chain/include/graphene/chain/protocol/types.hpp index af8a2ace..17b4049c 100644 --- a/libraries/chain/include/graphene/chain/protocol/types.hpp +++ b/libraries/chain/include/graphene/chain/protocol/types.hpp @@ -140,7 +140,6 @@ namespace graphene { namespace chain { impl_transaction_object_type, impl_block_summary_object_type, impl_account_transaction_history_object_type, - impl_witness_schedule_object_type, impl_blinded_balance_object_type, impl_chain_property_object_type }; @@ -165,7 +164,6 @@ namespace graphene { namespace chain { class operation_history_object; class withdraw_permission_object; class vesting_balance_object; - class witness_schedule_object; class worker_object; class balance_object; @@ -209,7 +207,6 @@ namespace graphene { namespace chain { typedef object_id< implementation_ids, impl_account_transaction_history_object_type, account_transaction_history_object> account_transaction_history_id_type; - typedef object_id< implementation_ids, impl_witness_schedule_object_type, witness_schedule_object > witness_schedule_id_type; typedef object_id< implementation_ids, impl_chain_property_object_type, chain_property_object> chain_property_id_type; typedef fc::array symbol_type; @@ -286,7 +283,6 @@ FC_REFLECT_ENUM( graphene::chain::impl_object_type, (impl_transaction_object_type) (impl_block_summary_object_type) (impl_account_transaction_history_object_type) - (impl_witness_schedule_object_type) (impl_blinded_balance_object_type) (impl_chain_property_object_type) ) @@ -317,7 +313,6 @@ FC_REFLECT_TYPENAME( graphene::chain::account_statistics_id_type ) FC_REFLECT_TYPENAME( graphene::chain::transaction_obj_id_type ) FC_REFLECT_TYPENAME( graphene::chain::block_summary_id_type ) FC_REFLECT_TYPENAME( graphene::chain::account_transaction_history_id_type ) -FC_REFLECT_TYPENAME( graphene::chain::witness_schedule_id_type ) FC_REFLECT( graphene::chain::void_t, ) FC_REFLECT_ENUM( graphene::chain::asset_issuer_permission_flags, (charge_market_fee)(white_list)(transfer_restricted)(override_authority)(disable_force_settle)(global_settle)(disable_confidential) ) diff --git a/libraries/chain/include/graphene/chain/witness_object.hpp b/libraries/chain/include/graphene/chain/witness_object.hpp index 17b94ed5..0576df6c 100644 --- a/libraries/chain/include/graphene/chain/witness_object.hpp +++ b/libraries/chain/include/graphene/chain/witness_object.hpp @@ -32,6 +32,7 @@ namespace graphene { namespace chain { static const uint8_t type_id = witness_object_type; account_id_type witness_account; + uint64_t last_aslot = 0; public_key_type signing_key; secret_hash_type next_secret_hash; secret_hash_type previous_secret; @@ -64,9 +65,8 @@ namespace graphene { namespace chain { FC_REFLECT_DERIVED( graphene::chain::witness_object, (graphene::db::object), (witness_account) + (last_aslot) (signing_key) - (next_secret_hash) - (previous_secret) (pay_vb) (vote_id) (total_votes) diff --git a/libraries/chain/include/graphene/chain/witness_schedule_object.hpp b/libraries/chain/include/graphene/chain/witness_schedule_object.hpp deleted file mode 100644 index cd11ebca..00000000 --- a/libraries/chain/include/graphene/chain/witness_schedule_object.hpp +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2015, Cryptonomex, Inc. - * All rights reserved. - * - * This source code is provided for evaluation in private test networks only, until September 8, 2015. After this date, this license expires and - * the code may not be used, modified or distributed for any purpose. Redistribution and use in source and binary forms, with or without modification, - * are permitted until September 8, 2015, provided that the following conditions are met: - * - * 1. The code and/or derivative works are used only for private test networks consisting of no more than 10 P2P nodes. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -#pragma once - -// needed to serialize witness_scheduler -#include -#include - -#include -#include -#include - -namespace graphene { namespace chain { - -typedef hash_ctr_rng< - /* HashClass = */ fc::sha256, - /* SeedLength = */ GRAPHENE_RNG_SEED_LENGTH - > witness_scheduler_rng; - -typedef generic_witness_scheduler< - /* WitnessID = */ witness_id_type, - /* RNG = */ witness_scheduler_rng, - /* CountType = */ decltype( chain_parameters::maximum_witness_count ), - /* OffsetType = */ uint32_t, - /* debug = */ true - > witness_scheduler; - -typedef generic_far_future_witness_scheduler< - /* WitnessID = */ witness_id_type, - /* RNG = */ witness_scheduler_rng, - /* CountType = */ decltype( chain_parameters::maximum_witness_count ), - /* OffsetType = */ uint32_t, - /* debug = */ true - > far_future_witness_scheduler; - -class witness_schedule_object : public abstract_object -{ - public: - static const uint8_t space_id = implementation_ids; - static const uint8_t type_id = impl_witness_schedule_object_type; - - witness_scheduler scheduler; - uint32_t last_scheduling_block; - uint64_t slots_since_genesis = 0; - fc::array< char, sizeof(secret_hash_type) > rng_seed; - - /** - * Not necessary for consensus, but used for figuring out the participation rate. - * The nth bit is 0 if the nth slot was unfilled, else it is 1. - */ - fc::uint128 recent_slots_filled; -}; - -} } - -FC_REFLECT( graphene::chain::witness_scheduler, - (_turns) - (_tokens) - (_min_token_count) - (_ineligible_waiting_for_token) - (_ineligible_no_turn) - (_eligible) - (_schedule) - (_lame_duck) - ) - -FC_REFLECT_DERIVED( graphene::chain::witness_schedule_object, (graphene::chain::object), - (scheduler) - (last_scheduling_block) - (slots_since_genesis) - (rng_seed) - (recent_slots_filled) - ) diff --git a/libraries/chain/include/graphene/chain/witness_scheduler.hpp b/libraries/chain/include/graphene/chain/witness_scheduler.hpp deleted file mode 100644 index 5feef417..00000000 --- a/libraries/chain/include/graphene/chain/witness_scheduler.hpp +++ /dev/null @@ -1,417 +0,0 @@ -/* - * Copyright (c) 2015, Cryptonomex, Inc. - * All rights reserved. - * - * This source code is provided for evaluation in private test networks only, until September 8, 2015. After this date, this license expires and - * the code may not be used, modified or distributed for any purpose. Redistribution and use in source and binary forms, with or without modification, - * are permitted until September 8, 2015, provided that the following conditions are met: - * - * 1. The code and/or derivative works are used only for private test networks consisting of no more than 10 P2P nodes. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -#pragma once - -#include -#include -#include -#include - -#include - -namespace graphene { namespace chain { - -//using boost::container::flat_set; - -enum witness_scheduler_relax_flags -{ - emit_turn = 0x01, - emit_token = 0x02 -}; - -template< typename WitnessID, typename RNG, typename CountType, typename OffsetType, bool debug = true > -class generic_witness_scheduler -{ - public: - void check_invariant() const - { -#ifndef NDEBUG - CountType tokens = _ineligible_no_turn.size() + _eligible.size(); - CountType turns = _eligible.size(); - for( const std::pair< WitnessID, bool >& item : _ineligible_waiting_for_token ) - turns += (item.second ? 1 : 0 ); - - assert( _tokens == tokens ); - assert( _turns == turns ); -#endif - - set< WitnessID > witness_set; - // make sure each witness_id occurs only once among the three states - auto process_id = [&]( WitnessID item ) - { - assert( witness_set.find( item ) == witness_set.end() ); - witness_set.insert( item ); - } ; - - for( const std::pair< WitnessID, bool >& item : _ineligible_waiting_for_token ) - process_id( item.first ); - for( const WitnessID& item : _ineligible_no_turn ) - process_id( item ); - for( const WitnessID& item : _eligible ) - process_id( item ); - return; - } - - /** - * Deterministically evolve over time - */ - uint32_t relax() - { - uint32_t relax_flags = 0; - - if( debug ) check_invariant(); - assert( _min_token_count > 0 ); - - // turn distribution - if( _turns == 0 ) - { - relax_flags |= emit_turn; - for( const WitnessID& item : _ineligible_no_turn ) - _eligible.push_back( item ); - _turns += _ineligible_no_turn.size(); - _ineligible_no_turn.clear(); - if( debug ) check_invariant(); - - for( std::pair< WitnessID, bool >& item : _ineligible_waiting_for_token ) - { - assert( item.second == false ); - item.second = true; - } - _turns += _ineligible_waiting_for_token.size(); - if( debug ) check_invariant(); - } - - // token distribution - while( true ) - { - if( _ineligible_waiting_for_token.empty() ) - { - // eligible must be non-empty - assert( !_eligible.empty() ); - return relax_flags; - } - - if( _tokens >= _min_token_count ) - { - if( !_eligible.empty() ) - return relax_flags; - } - - const std::pair< WitnessID, bool >& item = _ineligible_waiting_for_token.front(); - if( item.second ) - _eligible.push_back( item.first ); - else - _ineligible_no_turn.push_back( item.first ); - _ineligible_waiting_for_token.pop_front(); - relax_flags |= emit_token; - _tokens++; - if( debug ) check_invariant(); - } - - return relax_flags; - } - - /** - * Add another element to _schedule - */ - uint32_t produce_schedule( RNG& rng ) - { - uint32_t relax_flags = relax(); - if( debug ) check_invariant(); - if( _eligible.empty() ) - return relax_flags; - - decltype( rng( _eligible.size() ) ) pos = rng( _eligible.size() ); - assert( (pos >= 0) && (pos < _eligible.size()) ); - auto it = _eligible.begin() + pos; - _schedule.push_back( *it ); - _ineligible_waiting_for_token.emplace_back( *it, false ); - _eligible.erase( it ); - _turns--; - _tokens--; - if( debug ) check_invariant(); - return relax_flags; - } - - /** - * Pull an element from _schedule - */ - WitnessID consume_schedule() - { - assert( _schedule.size() > 0 ); - - WitnessID result = _schedule.front(); - _schedule.pop_front(); - - auto it = _lame_duck.find( result ); - if( it != _lame_duck.end() ) - _lame_duck.erase( it ); - if( debug ) check_invariant(); - return result; - } - - /** - * Remove all witnesses in the removal_set from - * future scheduling (but not from the current schedule). - */ - template< typename T > - void remove_all( const T& removal_set ) - { - if( debug ) check_invariant(); - - _ineligible_waiting_for_token.erase( - std::remove_if( - _ineligible_waiting_for_token.begin(), - _ineligible_waiting_for_token.end(), - [&]( const std::pair< WitnessID, bool >& item ) -> bool - { - bool found = removal_set.find( item.first ) != removal_set.end(); - _turns -= (found & item.second) ? 1 : 0; - return found; - } ), - _ineligible_waiting_for_token.end() ); - if( debug ) check_invariant(); - - _ineligible_no_turn.erase( - std::remove_if( - _ineligible_no_turn.begin(), - _ineligible_no_turn.end(), - [&]( WitnessID item ) -> bool - { - bool found = (removal_set.find( item ) != removal_set.end()); - _tokens -= (found ? 1 : 0); - return found; - } ), - _ineligible_no_turn.end() ); - if( debug ) check_invariant(); - - _eligible.erase( - std::remove_if( - _eligible.begin(), - _eligible.end(), - [&]( WitnessID item ) -> bool - { - bool found = (removal_set.find( item ) != removal_set.end()); - _tokens -= (found ? 1 : 0); - _turns -= (found ? 1 : 0); - return found; - } ), - _eligible.end() ); - if( debug ) check_invariant(); - - return; - } - - /** - * Convenience function to call insert_all() and remove_all() - * as needed to update to the given revised_set. - */ - template< typename T > - void insert_all( const T& insertion_set ) - { - if( debug ) check_invariant(); - for( const WitnessID wid : insertion_set ) - { - _eligible.push_back( wid ); - } - _turns += insertion_set.size(); - _tokens += insertion_set.size(); - if( debug ) check_invariant(); - return; - } - - /** - * Convenience function to call insert_all() and remove_all() - * as needed to update to the given revised_set. - * - * This function calls find() on revised_set for all current - * witnesses. Running time is O(n*log(n)) if the revised_set - * implementation of find() is O(log(n)). - * - * TODO: Rewriting to use std::set_difference may marginally - * increase efficiency, but a benchmark is needed to justify this. - */ - template< typename T > - void update( const T& revised_set ) - { - set< WitnessID > current_set; - set< WitnessID > schedule_set; - - /* current_set.reserve( - _ineligible_waiting_for_token.size() - + _ineligible_no_turn.size() - + _eligible.size() - + _schedule.size() ); - */ - for( const auto& item : _ineligible_waiting_for_token ) - current_set.insert( item.first ); - for( const WitnessID& item : _ineligible_no_turn ) - current_set.insert( item ); - for( const WitnessID& item : _eligible ) - current_set.insert( item ); - for( const WitnessID& item : _schedule ) - { - current_set.insert( item ); - schedule_set.insert( item ); - } - - set< WitnessID > insertion_set; - //insertion_set.reserve( revised_set.size() ); - for( const WitnessID& item : revised_set ) - { - if( current_set.find( item ) == current_set.end() ) - insertion_set.insert( item ); - } - - set< WitnessID > removal_set; - //removal_set.reserve( current_set.size() ); - for( const WitnessID& item : current_set ) - { - if( revised_set.find( item ) == revised_set.end() ) - { - if( schedule_set.find( item ) == schedule_set.end() ) - removal_set.insert( item ); - else - _lame_duck.insert( item ); - } - } - - insert_all( insertion_set ); - remove_all( removal_set ); - - return; - } - - /** - * Get the number of scheduled witnesses - */ - - size_t size( )const - { - return _schedule.size(); - } - - bool get_slot( OffsetType offset, WitnessID& wit )const - { - if( offset >= _schedule.size() ) - return false; - wit = _schedule[ offset ]; - return true; - } - - /** - * Reset the schedule, then re-schedule the given witness as the - * first witness. - */ - void reset_schedule( WitnessID first_witness ) - { - _schedule.clear(); - for( const WitnessID& wid : _ineligible_no_turn ) - { - _eligible.push_back( wid ); - } - _turns += _ineligible_no_turn.size(); - _ineligible_no_turn.clear(); - for( const auto& item : _ineligible_waiting_for_token ) - { - _eligible.push_back( item.first ); - _turns += (item.second ? 0 : 1); - } - _tokens += _ineligible_waiting_for_token.size(); - _ineligible_waiting_for_token.clear(); - if( debug ) check_invariant(); - - auto it = std::find( _eligible.begin(), _eligible.end(), first_witness ); - assert( it != _eligible.end() ); - - _schedule.push_back( *it ); - _ineligible_waiting_for_token.emplace_back( *it, false ); - _eligible.erase( it ); - _turns--; - _tokens--; - if( debug ) check_invariant(); - return; - } - - // keep track of total turns / tokens in existence - CountType _turns = 0; - CountType _tokens = 0; - - // new tokens handed out when _tokens < _min_token_count - CountType _min_token_count; - - // WitnessID appears in exactly one of the following: - // has no token; second indicates whether we have a turn or not: - std::deque < std::pair< WitnessID, bool > > _ineligible_waiting_for_token; // ".." | "T." - // has token, but no turn - std::vector< WitnessID > _ineligible_no_turn; // ".t" - // has token and turn - std::vector< WitnessID > _eligible; // "Tt" - - // scheduled - std::deque < WitnessID > _schedule; - - // in _schedule, but not to be replaced - set< WitnessID > _lame_duck; -}; - -template< typename WitnessID, typename RNG, typename CountType, typename OffsetType, bool debug = true > -class generic_far_future_witness_scheduler -{ - public: - generic_far_future_witness_scheduler( - const generic_witness_scheduler< WitnessID, RNG, CountType, OffsetType, debug >& base_scheduler, - RNG rng - ) - { - generic_witness_scheduler< WitnessID, RNG, CountType, OffsetType, debug > extended_scheduler = base_scheduler; - _begin_offset = base_scheduler.size()+1; - while( (extended_scheduler.produce_schedule( rng ) & emit_turn) == 0 ) - _begin_offset++; - assert( _begin_offset == extended_scheduler.size() ); - - _end_offset = _begin_offset; - while( (extended_scheduler.produce_schedule( rng ) & emit_turn) == 0 ) - _end_offset++; - assert( _end_offset == extended_scheduler.size()-1 ); - _schedule.resize( extended_scheduler._schedule.size() ); - std::copy( extended_scheduler._schedule.begin(), - extended_scheduler._schedule.end(), - _schedule.begin() ); - return; - } - - bool get_slot( OffsetType offset, WitnessID& wit )const - { - if( offset <= _end_offset ) - wit = _schedule[ offset ]; - else - wit = _schedule[ _begin_offset + - ( - (offset - _begin_offset) % - (_end_offset + 1 - _begin_offset) - ) ]; - return true; - } - - std::vector< WitnessID > _schedule; - OffsetType _begin_offset; - OffsetType _end_offset; -}; - -} } diff --git a/libraries/chain/include/graphene/chain/witness_scheduler_rng.hpp b/libraries/chain/include/graphene/chain/witness_scheduler_rng.hpp deleted file mode 100644 index e7467010..00000000 --- a/libraries/chain/include/graphene/chain/witness_scheduler_rng.hpp +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (c) 2015, Cryptonomex, Inc. - * All rights reserved. - * - * This source code is provided for evaluation in private test networks only, until September 8, 2015. After this date, this license expires and - * the code may not be used, modified or distributed for any purpose. Redistribution and use in source and binary forms, with or without modification, - * are permitted until September 8, 2015, provided that the following conditions are met: - * - * 1. The code and/or derivative works are used only for private test networks consisting of no more than 10 P2P nodes. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -#pragma once - -#include - -namespace graphene { namespace chain { - -/** - * Always returns 0. Useful for testing. - */ -class nullary_rng -{ - public: - nullary_rng() {} - virtual ~nullary_rng() {} - - template< typename T > T operator()( T max ) - { return T(0); } -} ; - -/** - * The sha256_ctr_rng generates bits using SHA256 in counter (CTR) - * mode. - */ -template< class HashClass, int SeedLength=sizeof(secret_hash_type) > -class hash_ctr_rng -{ - public: - hash_ctr_rng( const char* seed, uint64_t counter = 0 ) - : _counter( counter ), _current_offset( 0 ) - { - memcpy( _seed, seed, SeedLength ); - _reset_current_value(); - return; - } - - virtual ~hash_ctr_rng() {} - - uint64_t get_bits( uint8_t count ) - { - uint64_t result = 0; - uint64_t mask = 1; - // grab the requested number of bits - while( count > 0 ) - { - result |= - ( - ( - ( - _current_value.data()[ (_current_offset >> 3) & 0x1F ] - & ( 1 << (_current_offset & 0x07) ) - ) - != 0 - ) ? mask : 0 - ); - mask += mask; - --count; - ++_current_offset; - if( _current_offset == (_current_value.data_size() << 3) ) - { - _counter++; - _current_offset = 0; - _reset_current_value(); - } - } - return result; - } - - uint64_t operator()( uint64_t bound ) - { - if( bound <= 1 ) - return 0; - uint8_t bitcount = boost::multiprecision::detail::find_msb( bound ) + 1; - - // probability of loop exiting is >= 1/2, so probability of - // running N times is bounded above by (1/2)^N - while( true ) - { - uint64_t result = get_bits( bitcount ); - if( result < bound ) - return result; - } - } - - // convenience method which does casting for types other than uint64_t - template< typename T > T operator()( T bound ) - { return (T) ( (*this)(uint64_t( bound )) ); } - - void _reset_current_value() - { - // internal implementation detail, called to update - // _current_value when _counter changes - typename HashClass::encoder enc; - enc.write( _seed , SeedLength ); - enc.write( (char *) &_counter, 8 ); - _current_value = enc.result(); - return; - } - - uint64_t _counter; - char _seed[ SeedLength ]; - HashClass _current_value; - uint16_t _current_offset; - - static const int seed_length = SeedLength; -} ; - -} } diff --git a/libraries/wallet/wallet.cpp b/libraries/wallet/wallet.cpp index f34dbdb7..b61c752c 100644 --- a/libraries/wallet/wallet.cpp +++ b/libraries/wallet/wallet.cpp @@ -478,7 +478,6 @@ public: result["chain_id"] = chain_props.chain_id; result["active_witnesses"] = global_props.active_witnesses; result["active_committee_members"] = global_props.active_committee_members; - result["entropy"] = dynamic_props.random; return result; } chain_property_object get_chain_properties() const diff --git a/programs/size_checker/main.cpp b/programs/size_checker/main.cpp index 5950f6aa..9ef4420a 100644 --- a/programs/size_checker/main.cpp +++ b/programs/size_checker/main.cpp @@ -85,6 +85,7 @@ int main( int argc, char** argv ) std::cout << "\n"; } std::cout << "]\n"; + std::cerr << "Size of block header: " << sizeof( block_header ) << " " << fc::raw::pack_size( block_header() ) << "\n"; } catch ( const fc::exception& e ){ edump((e.to_detail_string())); } idump((sizeof(signed_block))); diff --git a/tests/intense/block_tests.cpp b/tests/intense/block_tests.cpp index fd77132e..7e975537 100644 --- a/tests/intense/block_tests.cpp +++ b/tests/intense/block_tests.cpp @@ -25,7 +25,6 @@ #include #include #include -#include #include #include diff --git a/tests/tests/basic_tests.cpp b/tests/tests/basic_tests.cpp index 6aeae968..c0779013 100644 --- a/tests/tests/basic_tests.cpp +++ b/tests/tests/basic_tests.cpp @@ -23,7 +23,6 @@ #include #include -#include #include #include @@ -166,7 +165,6 @@ BOOST_AUTO_TEST_CASE( price_test ) BOOST_CHECK(dummy == dummy2); } - BOOST_AUTO_TEST_CASE( memo_test ) { try { memo_data m; @@ -186,134 +184,6 @@ BOOST_AUTO_TEST_CASE( memo_test ) BOOST_CHECK_EQUAL(m.get_message(receiver, sender.get_public_key()), "Hello, world!"); } FC_LOG_AND_RETHROW() } -BOOST_AUTO_TEST_CASE( witness_rng_test_bits ) -{ - try - { - const uint64_t COUNT = 131072; - const uint64_t HASH_SIZE = 32; - string ref_bits = ""; - ref_bits.reserve( COUNT * HASH_SIZE ); - static const char seed_data[] = "\xe3\xb0\xc4\x42\x98\xfc\x1c\x14\x9a\xfb\xf4\xc8\x99\x6f\xb9\x24\x27\xae\x41\xe4\x64\x9b\x93\x4c\xa4\x95\x99\x1b\x78\x52\xb8\x55"; - - for( uint64_t i=0; i> 0x08) & 0xFF) ); - enc.put( char((i >> 0x10) & 0xFF) ); - enc.put( char((i >> 0x18) & 0xFF) ); - enc.put( char((i >> 0x20) & 0xFF) ); - enc.put( char((i >> 0x28) & 0xFF) ); - enc.put( char((i >> 0x30) & 0xFF) ); - enc.put( char((i >> 0x38) & 0xFF) ); - - fc::sha256 result = enc.result(); - auto result_data = result.data(); - std::copy( result_data, result_data+HASH_SIZE, std::back_inserter( ref_bits ) ); - } - - fc::sha256 seed = fc::sha256::hash( string("") ); - // seed = sha256("") = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 - BOOST_CHECK( memcmp( seed.data(), seed_data, HASH_SIZE ) == 0 ); - - hash_ctr_rng< fc::sha256, 32 > test_rng(seed.data(), 0); - // python2 -c 'import hashlib; import struct; h = lambda x : hashlib.sha256(x).digest(); i = lambda x : struct.pack(" uint64_t - { - uint64_t result = 0; - uint64_t i = ref_get_bits_offset; - uint64_t mask = 1; - while( count > 0 ) - { - if( ref_bits[ i >> 3 ] & (1 << (i & 7)) ) - result |= mask; - mask += mask; - i++; - count--; - } - ref_get_bits_offset = i; - return result; - }; - - // use PRNG to decide between 0-64 bits - std::minstd_rand rng; - rng.seed( 9999 ); - std::uniform_int_distribution< uint16_t > bit_dist( 0, 64 ); - for( int i=0; i<10000; i++ ) - { - uint8_t bit_count = bit_dist( rng ); - uint64_t ref_bits = ref_get_bits( bit_count ); - uint64_t test_bits = test_rng.get_bits( bit_count ); - //std::cout << i << ": get(" << int(bit_count) << ") -> " << test_bits << " (expect " << ref_bits << ")\n"; - if( bit_count < 64 ) - { - BOOST_CHECK( ref_bits < (uint64_t( 1 ) << bit_count ) ); - BOOST_CHECK( test_bits < (uint64_t( 1 ) << bit_count ) ); - } - BOOST_CHECK( ref_bits == test_bits ); - if( ref_bits != test_bits ) - break; - } - - std::uniform_int_distribution< uint64_t > whole_dist( - 0, std::numeric_limits::max() ); - for( int i=0; i<10000; i++ ) - { - uint8_t bit_count = bit_dist( rng ); - uint64_t bound = whole_dist( rng ) & ((uint64_t(1) << bit_count) - 1); - //std::cout << "bound:" << bound << "\n"; - uint64_t rnum = test_rng( bound ); - //std::cout << "rnum:" << rnum << "\n"; - if( bound > 1 ) - { - BOOST_CHECK( rnum < bound ); - } - else - { - BOOST_CHECK( rnum == 0 ); - } - } - - } FC_LOG_AND_RETHROW() -} - BOOST_AUTO_TEST_CASE( exceptions ) { GRAPHENE_CHECK_THROW(FC_THROW_EXCEPTION(balance_claim_invalid_claim_amount, "Etc"), balance_claim_invalid_claim_amount); diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp index d8301856..e6677213 100644 --- a/tests/tests/block_tests.cpp +++ b/tests/tests/block_tests.cpp @@ -25,7 +25,6 @@ #include #include #include -#include #include @@ -644,7 +643,6 @@ BOOST_FIXTURE_TEST_CASE( maintenance_interval, database_fixture ) initial_properties.parameters.maximum_transaction_size); BOOST_CHECK_EQUAL(db.get_dynamic_global_properties().next_maintenance_time.sec_since_epoch(), db.head_block_time().sec_since_epoch() + db.get_global_properties().parameters.block_interval); - // shuffling is now handled by the witness_schedule_object. BOOST_CHECK(db.get_global_properties().active_witnesses == initial_properties.active_witnesses); BOOST_CHECK(db.get_global_properties().active_committee_members == initial_properties.active_committee_members); @@ -870,32 +868,6 @@ BOOST_FIXTURE_TEST_CASE( pop_block_twice, database_fixture ) } } -BOOST_FIXTURE_TEST_CASE( witness_scheduler_missed_blocks, database_fixture ) -{ try { - db.get_near_witness_schedule(); - generate_block(); - auto near_schedule = db.get_near_witness_schedule(); - - std::for_each(near_schedule.begin(), near_schedule.end(), [&](witness_id_type id) { - generate_block(0); - BOOST_CHECK(db.get_dynamic_global_properties().current_witness == id); - }); - - near_schedule = db.get_near_witness_schedule(); - generate_block(0, init_account_priv_key, 2); - BOOST_CHECK(db.get_dynamic_global_properties().current_witness == near_schedule[2]); - - near_schedule.erase(near_schedule.begin(), near_schedule.begin() + 3); - auto new_schedule = db.get_near_witness_schedule(); - new_schedule.erase(new_schedule.end() - 3, new_schedule.end()); - BOOST_CHECK(new_schedule == near_schedule); - - std::for_each(near_schedule.begin(), near_schedule.end(), [&](witness_id_type id) { - generate_block(0); - BOOST_CHECK(db.get_dynamic_global_properties().current_witness == id); - }); -} FC_LOG_AND_RETHROW() } - BOOST_FIXTURE_TEST_CASE( rsf_missed_blocks, database_fixture ) { try @@ -904,7 +876,7 @@ BOOST_FIXTURE_TEST_CASE( rsf_missed_blocks, database_fixture ) auto rsf = [&]() -> string { - fc::uint128 rsf = db.get( witness_schedule_id_type() ).recent_slots_filled; + fc::uint128 rsf = db.get_dynamic_global_properties().recent_slots_filled; string result = ""; result.reserve(128); for( int i=0; i<128; i++ ) diff --git a/tests/tests/operation_tests2.cpp b/tests/tests/operation_tests2.cpp index 59c34c92..925c8a8d 100644 --- a/tests/tests/operation_tests2.cpp +++ b/tests/tests/operation_tests2.cpp @@ -443,34 +443,15 @@ BOOST_AUTO_TEST_CASE( witness_create ) auto itr = std::find(witnesses.begin(), witnesses.end(), nathan_witness_id); BOOST_CHECK(itr != witnesses.end()); - generate_blocks(witnesses.size()); - - // make sure we're scheduled to produce - vector near_witnesses = db.get_near_witness_schedule(); - BOOST_CHECK( std::find( near_witnesses.begin(), near_witnesses.end(), nathan_witness_id ) - != near_witnesses.end() ); - - struct generator_helper { - database_fixture& f; - witness_id_type nathan_id; - fc::ecc::private_key nathan_key; - bool nathan_generated_block; - - void operator()(witness_id_type id) { - if( id == nathan_id ) - { - nathan_generated_block = true; - f.generate_block(0, nathan_key); - } else - f.generate_block(0); - BOOST_CHECK_EQUAL(f.db.get_dynamic_global_properties().current_witness.instance.value, id.instance.value); - f.db.get_near_witness_schedule(); - } - }; - - generator_helper h = std::for_each(near_witnesses.begin(), near_witnesses.end(), - generator_helper{*this, nathan_witness_id, nathan_private_key, false}); - BOOST_CHECK(h.nathan_generated_block); + int produced = 0; + // Make sure we get scheduled exactly once in witnesses.size() blocks + // TODO: intense_test that repeats this loop many times + for( size_t i=0; i Date: Wed, 26 Aug 2015 18:01:48 -0400 Subject: [PATCH 4/8] update subscribe callback --- libraries/app/api.cpp | 164 ++++++++---------- libraries/app/include/graphene/app/api.hpp | 42 ++--- .../delayed_node/delayed_node_plugin.cpp | 20 ++- libraries/wallet/wallet.cpp | 15 +- programs/CMakeLists.txt | 2 +- 5 files changed, 100 insertions(+), 143 deletions(-) diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index ef7f3a13..e851258c 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -52,8 +52,44 @@ namespace graphene { namespace app { elog("freeing database api ${x}", ("x",int64_t(this)) ); } + void database_api::set_subscribe_callback( std::function cb, bool clear_filter ) + { + edump((clear_filter)); + _subscribe_callback = cb; + if( clear_filter || !cb ) + { + static fc::bloom_parameters param; + param.projected_element_count = 10000; + param.false_positive_probability = 1.0/10000; + param.maximum_size = 1024*8*8*2; + _subscribe_filter = fc::bloom_filter(param); + } + } + + void database_api::subscribe_to_id( object_id_type id )const + { + idump((id)); + if( _subscribe_callback ) + _subscribe_filter.insert( (const unsigned char*)&id, sizeof(id) ); + else + elog( "unable to subscribe to id because there is no subscribe callback set" ); + } fc::variants database_api::get_objects(const vector& ids)const { + if( _subscribe_callback ) { + for( auto id : ids ) + { + if( id.type() == operation_history_object_type && id.space() == protocol_ids ) continue; + if( id.type() == impl_account_transaction_history_object_type && id.space() == implementation_ids ) continue; + + subscribe_to_id( id ); + } + } + else + { + elog( "getObjects without subscribe callback??" ); + } + fc::variants result; result.reserve(ids.size()); @@ -150,7 +186,10 @@ namespace graphene { namespace app { std::transform(account_ids.begin(), account_ids.end(), std::back_inserter(result), [this](account_id_type id) -> optional { if(auto o = _db.find(id)) + { + subscribe_to_id( id ); return *o; + } return {}; }); return result; @@ -162,7 +201,10 @@ namespace graphene { namespace app { std::transform(asset_ids.begin(), asset_ids.end(), std::back_inserter(result), [this](asset_id_type id) -> optional { if(auto o = _db.find(id)) + { + subscribe_to_id( id ); return *o; + } return {}; }); return result; @@ -182,35 +224,17 @@ namespace graphene { namespace app { for( auto itr = accounts_by_name.lower_bound(lower_bound_name); limit-- && itr != accounts_by_name.end(); ++itr ) + { result.insert(make_pair(itr->name, itr->get_id())); + if( limit == 1 ) + subscribe_to_id( itr->get_id() ); + } return result; } - void database_api::unsubscribe_from_accounts( const vector& names_or_ids ) - { - for (const std::string& account_name_or_id : names_or_ids) - { - const account_object* account = nullptr; - if (std::isdigit(account_name_or_id[0])) - account = _db.find(fc::variant(account_name_or_id).as()); - else - { - const auto& idx = _db.get_index_type().indices().get(); - auto itr = idx.find(account_name_or_id); - if (itr != idx.end()) - account = &*itr; - } - if (account == nullptr) - continue; - _account_subscriptions.erase(account->id); - } - } - - std::map database_api::get_full_accounts(std::function callback, - const vector& names_or_ids, bool subscribe) + std::map database_api::get_full_accounts( const vector& names_or_ids, bool subscribe) { - FC_ASSERT( _account_subscriptions.size() < 1024 ); std::map results; for (const std::string& account_name_or_id : names_or_ids) @@ -231,12 +255,7 @@ namespace graphene { namespace app { if( subscribe ) { ilog( "subscribe to ${id}", ("id",account->name) ); - _account_subscriptions[account->id] = callback; - } - else - { - wlog( "unsubscribe to ${id}", ("id",account->name) ); - _account_subscriptions.erase(account->id); + subscribe_to_id( account->id ); } // fc::mutable_variant_object full_account; @@ -795,7 +814,7 @@ namespace graphene { namespace app { /// we need to ensure the database_api is not deleted for the life of the async operation auto capture_this = shared_from_this(); - if( _account_subscriptions.size() ) + if( _subscribe_callback ) { map > broadcast_queue; for( const auto& obj : objs ) @@ -803,24 +822,21 @@ namespace graphene { namespace app { auto relevant = get_relevant_accounts( obj ); for( const auto& r : relevant ) { - auto sub = _account_subscriptions.find(r); - if( sub != _account_subscriptions.end() ) + if( _subscribe_filter.contains(r) ) broadcast_queue[r].emplace_back(obj->to_variant()); } + if( relevant.size() == 0 && _subscribe_filter.contains(obj->id) ) + broadcast_queue[account_id_type()].emplace_back(obj->to_variant()); } if( broadcast_queue.size() ) { fc::async([capture_this,broadcast_queue,this](){ - for( const auto& item : broadcast_queue ) - { - auto sub = _account_subscriptions.find(item.first); - if( sub != _account_subscriptions.end() ) - sub->second( fc::variant(item.second ) ); - } + _subscribe_callback( fc::variant(broadcast_queue) ); }); } } + if( _market_subscriptions.size() ) { map< pair, vector > broadcast_queue; @@ -850,16 +866,14 @@ namespace graphene { namespace app { void database_api::on_objects_changed(const vector& ids) { - vector my_objects; - map > broadcast_queue; + vector updates; map< pair, vector > market_broadcast_queue; + + idump((ids)); for(auto id : ids) { - if(_subscriptions.find(id) != _subscriptions.end()) - my_objects.push_back(id); - const object* obj = nullptr; - if( _account_subscriptions.size() ) + if( _subscribe_callback ) { obj = _db.find_object( id ); if( obj ) @@ -867,18 +881,23 @@ namespace graphene { namespace app { vector relevant = get_relevant_accounts( obj ); for( const auto& r : relevant ) { - auto sub = _account_subscriptions.find(r); - if( sub != _account_subscriptions.end() ) - broadcast_queue[r].emplace_back(obj->to_variant()); + if( _subscribe_filter.contains(r) ) + updates.emplace_back(obj->to_variant()); } + if( relevant.size() == 0 && _subscribe_filter.contains(obj->id) ) + updates.emplace_back(obj->to_variant()); } else - elog( "unable to find object ${id}", ("id",id) ); + { + if( _subscribe_filter.contains(id) ) + updates.emplace_back(id); // send just the id to indicate removal + } } if( _market_subscriptions.size() ) { - if( !_account_subscriptions.size() ) obj = _db.find_object( id ); + if( !_subscribe_callback ) + obj = _db.find_object( id ); if( obj ) { const limit_order_object* order = dynamic_cast(obj); @@ -897,42 +916,15 @@ namespace graphene { namespace app { /// pushing the future back / popping the prior future if it is complete. /// if a connection hangs then this could get backed up and result in /// a failure to exit cleanly. - fc::async([capture_this,this,broadcast_queue,market_broadcast_queue,my_objects](){ - for( const auto& item : broadcast_queue ) - { - edump( (item) ); - try { - auto sub = _account_subscriptions.find(item.first); - if( sub != _account_subscriptions.end() ) - sub->second( fc::variant(item.second ) ); - } catch ( const fc::exception& e ) - { - edump((e.to_detail_string())); - } - } + fc::async([capture_this,this,updates,market_broadcast_queue](){ + if( _subscribe_callback ) _subscribe_callback( updates ); + for( const auto& item : market_broadcast_queue ) { auto sub = _market_subscriptions.find(item.first); if( sub != _market_subscriptions.end() ) sub->second( fc::variant(item.second ) ); } - for(auto id : my_objects) - { - // just incase _usbscriptions changed between filter and broadcast - auto itr = _subscriptions.find( id ); - if( itr != _subscriptions.end() ) - { - const object* obj = _db.find_object( id ); - if( obj != nullptr ) - { - itr->second(obj->to_variant()); - } - else - { - itr->second(fc::variant(id)); - } - } - } }); } @@ -980,20 +972,6 @@ namespace graphene { namespace app { }); } - - vector database_api::subscribe_to_objects( const std::function& callback, const vector& ids) - { - FC_ASSERT( _subscriptions.size() < 1024 ); - for(auto id : ids) _subscriptions[id] = callback; - return get_objects( ids ); - } - - bool database_api::unsubscribe_from_objects(const vector& ids) - { - for(auto id : ids) _subscriptions.erase(id); - return true; - } - void database_api::subscribe_to_market(std::function callback, asset_id_type a, asset_id_type b) { if(a > b) std::swap(a,b); diff --git a/libraries/app/include/graphene/app/api.hpp b/libraries/app/include/graphene/app/api.hpp index 04ad5cfb..0d459bf5 100644 --- a/libraries/app/include/graphene/app/api.hpp +++ b/libraries/app/include/graphene/app/api.hpp @@ -40,6 +40,7 @@ #include #include +#include namespace graphene { namespace app { using namespace graphene::chain; @@ -176,13 +177,8 @@ namespace graphene { namespace app { * ignored. All other accounts will be retrieved and subscribed. * */ - std::map get_full_accounts(std::function callback, - const vector& names_or_ids, bool subscribe ); + std::map get_full_accounts( const vector& names_or_ids, bool subscribe ); - /** - * Stop receiving updates generated by get_full_accounts() - */ - void unsubscribe_from_accounts( const vector& names_or_ids ); /** * @brief Get limit orders in a given market @@ -277,24 +273,6 @@ namespace graphene { namespace app { */ vector> get_committee_members(const vector& committee_member_ids)const; - /** - * @group Push Notification Methods - * These methods may be used to get push notifications whenever an object or market is changed - * @{ - */ - /** - * @brief Request notifications when some object(s) change - * @param callback Callback method which is called with the new version of a changed object - * @param ids The set of object IDs to watch - * @return get_objects(ids) - */ - vector subscribe_to_objects(const std::function& callback, - const vector& ids); - /** - * @brief Stop receiving notifications for some object(s) - * @param ids The set of object IDs to stop watching - */ - bool unsubscribe_from_objects(const vector& ids); /** * @brief Request notification when the active orders in the market between two assets changes * @param callback Callback method which is called when the market changes @@ -318,7 +296,7 @@ namespace graphene { namespace app { * This unsubscribes from all subscribed markets and objects. */ void cancel_all_subscriptions() - { _subscriptions.clear(); _market_subscriptions.clear(); } + { set_subscribe_callback( std::function(), true); _market_subscriptions.clear(); } ///@} /// @brief Get a hexdump of the serialized binary form of a transaction @@ -377,18 +355,22 @@ namespace graphene { namespace app { */ vector get_required_fees( const vector& ops, asset_id_type id = asset_id_type() )const; + void set_subscribe_callback( std::function cb, bool clear_filter ); private: + void subscribe_to_id( object_id_type id )const; + /** called every time a block is applied to report the objects that were changed */ void on_objects_changed(const vector& ids); void on_objects_removed(const vector& objs); void on_applied_block(); + mutable fc::bloom_filter _subscribe_filter; + std::function _subscribe_callback; + boost::signals2::scoped_connection _change_connection; boost::signals2::scoped_connection _removed_connection; boost::signals2::scoped_connection _applied_block_connection; - map > _subscriptions; - map > _account_subscriptions; - map< pair, std::function > _market_subscriptions; + map< pair, std::function > _market_subscriptions; graphene::chain::database& _db; }; @@ -561,7 +543,6 @@ FC_API(graphene::app::database_api, (get_account_count) (lookup_accounts) (get_full_accounts) - (unsubscribe_from_accounts) (get_account_balances) (get_named_account_balances) (lookup_asset_symbols) @@ -577,8 +558,6 @@ FC_API(graphene::app::database_api, (get_witness_count) (lookup_witness_accounts) (lookup_committee_member_accounts) - (subscribe_to_objects) - (unsubscribe_from_objects) (subscribe_to_market) (unsubscribe_from_market) (cancel_all_subscriptions) @@ -594,6 +573,7 @@ FC_API(graphene::app::database_api, (verify_authority) (get_blinded_balances) (get_required_fees) + (set_subscribe_callback) ) FC_API(graphene::app::history_api, (get_account_history) diff --git a/libraries/plugins/delayed_node/delayed_node_plugin.cpp b/libraries/plugins/delayed_node/delayed_node_plugin.cpp index d0461a39..be5cbe2a 100644 --- a/libraries/plugins/delayed_node/delayed_node_plugin.cpp +++ b/libraries/plugins/delayed_node/delayed_node_plugin.cpp @@ -97,14 +97,26 @@ void delayed_node_plugin::plugin_startup() try { connect(); + my->database_api->set_subscribe_callback([this] (const fc::variant& v) { + auto& updates = v.get_array(); + for( const auto& v : updates ) + { + if( v.is_object() ) + { + auto& obj = v.get_object(); + if( obj["id"].as() == graphene::chain::dynamic_global_property_id_type() ) + { + auto props = v.as(); + sync_with_trusted_node(props.head_block_number); + } + } + } + }, true); + // Go ahead and get in sync now, before subscribing chain::dynamic_global_property_object props = my->database_api->get_dynamic_global_properties(); sync_with_trusted_node(props.head_block_number); - my->database_api->subscribe_to_objects([this] (const fc::variant& v) { - auto props = v.as(); - sync_with_trusted_node(props.head_block_number); - }, {graphene::chain::dynamic_global_property_id_type()}); return; } catch (const fc::exception& e) { elog("Error during connection: ${e}", ("e", e.to_detail_string())); diff --git a/libraries/wallet/wallet.cpp b/libraries/wallet/wallet.cpp index b61c752c..cf9ab236 100644 --- a/libraries/wallet/wallet.cpp +++ b/libraries/wallet/wallet.cpp @@ -375,10 +375,6 @@ public: ("chain_id", _chain_id) ); } init_prototype_ops(); - _remote_db->subscribe_to_objects( [=]( const fc::variant& obj ) - { - fc::async([this]{resync();}, "Resync after block"); - }, {dynamic_global_property_id_type()} ); _wallet.chain_id = _chain_id; _wallet.ws_server = initial_data.ws_server; _wallet.ws_user = initial_data.ws_user; @@ -629,10 +625,7 @@ public: _keys[wif_pub_key] = wif_key; - if( _wallet.update_account(account) ) - _remote_db->subscribe_to_objects([this](const fc::variant& v) { - _wallet.update_account(v.as()); - }, {account.id}); + _wallet.update_account(account); _wallet.extra_keys[account.id].insert(wif_pub_key); @@ -649,17 +642,11 @@ public: if( ! fc::exists( wallet_filename ) ) return false; - if( !_wallet.my_accounts.empty() ) - _remote_db->unsubscribe_from_objects(_wallet.my_account_ids()); _wallet = fc::json::from_file( wallet_filename ).as< wallet_data >(); if( _wallet.chain_id != _chain_id ) FC_THROW( "Wallet chain ID does not match", ("wallet.chain_id", _wallet.chain_id) ("chain_id", _chain_id) ); - if( !_wallet.my_accounts.empty() ) - _remote_db->subscribe_to_objects([this](const fc::variant& v) { - _wallet.update_account(v.as()); - }, _wallet.my_account_ids()); return true; } void save_wallet_file(string wallet_filename = "") diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 75bb6af3..ab267cb1 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -6,7 +6,7 @@ add_subdirectory( size_checker ) set(BUILD_QT_GUI FALSE CACHE BOOL "Build the Qt-based light client GUI") if(BUILD_QT_GUI) - add_subdirectory(light_client) +# add_subdirectory(light_client) endif() set(BUILD_WEB_NODE FALSE CACHE BOOL "Build the Qt-based full node with web GUI") if(BUILD_WEB_NODE) From 761fcb3d20d7c1a5e1f53dc2672ca5fa9212c7b5 Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Wed, 26 Aug 2015 18:45:06 -0400 Subject: [PATCH 5/8] fix bloom filter to set optimal parameters --- libraries/app/api.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index 18b3311d..13df7dfc 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -62,6 +62,7 @@ namespace graphene { namespace app { param.projected_element_count = 10000; param.false_positive_probability = 1.0/10000; param.maximum_size = 1024*8*8*2; + param.compute_optimal_parameters(); _subscribe_filter = fc::bloom_filter(param); } } From df3318efc6d6bbde5aee9b9ab4c5fe264c60a5af Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Thu, 27 Aug 2015 09:08:38 -0400 Subject: [PATCH 6/8] adding extra checks for unusual failure of get_scheduled_witness --- libraries/app/api.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index 13df7dfc..b0fab7fe 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -823,7 +823,10 @@ namespace graphene { namespace app { for( const auto& r : relevant ) { if( _subscribe_filter.contains(r) ) + { broadcast_queue[r].emplace_back(obj->to_variant()); + break; + } } if( relevant.size() == 0 && _subscribe_filter.contains(obj->id) ) broadcast_queue[account_id_type()].emplace_back(obj->to_variant()); @@ -882,7 +885,10 @@ namespace graphene { namespace app { for( const auto& r : relevant ) { if( _subscribe_filter.contains(r) ) + { updates.emplace_back(obj->to_variant()); + break; + } } if( relevant.size() == 0 && _subscribe_filter.contains(obj->id) ) updates.emplace_back(obj->to_variant()); From b8e16e2e9468f071dac871581beb342bd58457cc Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Thu, 27 Aug 2015 10:45:44 -0400 Subject: [PATCH 7/8] fix object id math --- libraries/db/include/graphene/db/object_id.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libraries/db/include/graphene/db/object_id.hpp b/libraries/db/include/graphene/db/object_id.hpp index 84428c7e..b195e8c3 100644 --- a/libraries/db/include/graphene/db/object_id.hpp +++ b/libraries/db/include/graphene/db/object_id.hpp @@ -54,6 +54,9 @@ namespace graphene { namespace db { object_id_type& operator++(int) { ++number; return *this; } object_id_type& operator++() { ++number; return *this; } + friend object_id_type operator+(const object_id a, int delta ) { + return object_id_type( space(), type(), instance() + delta ); + } friend size_t hash_value( object_id_type v ) { return std::hash()(v.number); } friend bool operator < ( const object_id_type& a, const object_id_type& b ) @@ -83,6 +86,9 @@ namespace graphene { namespace db { object_id( object_id_type id ):instance(id.instance()) { } + + friend object_id operator+(const object_id a, int delta ) { return object_id( instance+delta ); } + operator object_id_type()const { return object_id_type( SpaceID, TypeID, instance.value ); } operator uint64_t()const { return object_id_type( *this ).number; } From 3bc66e31a21d160073648f3dcef92266a975ea82 Mon Sep 17 00:00:00 2001 From: Daniel Larimer Date: Thu, 27 Aug 2015 10:46:09 -0400 Subject: [PATCH 8/8] potential fix and extra debugging for get_scheduled_witness --- libraries/chain/db_witness_schedule.cpp | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/libraries/chain/db_witness_schedule.cpp b/libraries/chain/db_witness_schedule.cpp index d12bf69e..2174a19f 100644 --- a/libraries/chain/db_witness_schedule.cpp +++ b/libraries/chain/db_witness_schedule.cpp @@ -45,12 +45,11 @@ witness_id_type database::get_scheduled_witness( uint32_t slot_num )const const flat_set< witness_id_type >& active_witnesses = get_global_properties().active_witnesses; uint32_t n = active_witnesses.size(); - uint64_t min_witness_separation = (n / 2)+1; + uint64_t min_witness_separation = (n / 2); /// should work in cases where n is 0,1, and 2 uint64_t current_aslot = get_dynamic_global_properties().current_aslot + slot_num; uint64_t start_of_current_round_aslot = current_aslot - (current_aslot % n); - uint64_t first_ineligible_aslot = std::min( - start_of_current_round_aslot, current_aslot - min_witness_separation ); + uint64_t first_ineligible_aslot = std::min( start_of_current_round_aslot + 1, current_aslot - min_witness_separation ); // // overflow analysis of above subtraction: // @@ -76,7 +75,9 @@ witness_id_type database::get_scheduled_witness( uint32_t slot_num )const if( wit.last_aslot >= first_ineligible_aslot ) continue; - uint64_t k = now_hi + uint64_t(wit_id); + /// High performance random generator + /// http://xorshift.di.unimi.it/ + uint64_t k = now_hi + uint64_t(wit_id)*2685821657736338717ULL; k ^= (k >> 12); k ^= (k << 25); k ^= (k >> 27); @@ -93,8 +94,17 @@ witness_id_type database::get_scheduled_witness( uint32_t slot_num )const // at most K elements are susceptible to the filter, // otherwise we have an inconsistent database (such as // wit.last_aslot values that are non-unique or in the future) + if( !success ) { + edump((best_k)(slot_num)(first_ineligible_aslot)(current_aslot)(start_of_current_round_aslot)(min_witness_separation)(active_witnesses.size())); - assert( success ); + for( const witness_id_type& wit_id : active_witnesses ) + { + const witness_object& wit = wit_id(*this); + if( wit.last_aslot >= first_ineligible_aslot ) + idump((wit_id)(wit.last_aslot)); + } + assert( success ); + } return best_wit; }