diff --git a/README.md b/README.md index 0fa53eb2..fa374b92 100644 --- a/README.md +++ b/README.md @@ -165,29 +165,7 @@ it is fairly simple to write API methods to expose database methods. Running private testnet ----------------------- -Normally `witness_node` assumes it won't be producing blocks from -genesis, or against very old chain state. We need to get `witness_node` -to discard this assumption if we actually want to start a new chain, -so we will need to specify in `config.ini`: - - enable-stale-production = true - -We also need to specify which witnesses will produce blocks locally; -`witness_node` does not assume that it should produce blocks for a given -witness just because it has the correct private key to do so. There are -ten witnesses at genesis of the testnet, block production can be -enabled for all of them by specifying multiple times in `config.ini`: - - witness-id = "1.6.0" - witness-id = "1.6.1" - witness-id = "1.6.2" - witness-id = "1.6.3" - witness-id = "1.6.4" - witness-id = "1.6.5" - witness-id = "1.6.6" - witness-id = "1.6.7" - witness-id = "1.6.8" - witness-id = "1.6.9" +See the [documentation](https://github.com/cryptonomex/graphene/wiki/private-testnet) if you want to run a private testnet. Questions --------- diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index 5fc139fe..e440854f 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -45,6 +45,10 @@ namespace graphene { namespace app { on_objects_removed(objs); }); _applied_block_connection = _db.applied_block.connect([this](const signed_block&){ on_applied_block(); }); + + _pending_trx_connection = _db.on_pending_transaction.connect([this](const signed_transaction& trx ){ + if( _pending_trx_callback ) _pending_trx_callback( fc::variant(trx) ); + }); } database_api::~database_api() @@ -629,6 +633,12 @@ namespace graphene { namespace app { _app.p2p_node()->broadcast_transaction(trx); } + void network_broadcast_api::broadcast_block( const signed_block& b ) + { + _app.chain_database()->push_block(b); + _app.p2p_node()->broadcast( net::block_message( b )); + } + void network_broadcast_api::broadcast_transaction_with_callback(confirmation_callback cb, const signed_transaction& trx) { trx.validate(); @@ -913,10 +923,18 @@ namespace graphene { namespace app { */ void database_api::on_applied_block() { + if (_block_applied_callback) + { + auto capture_this = shared_from_this(); + block_id_type block_id = _db.head_block_id(); + fc::async([this,capture_this,block_id](){ + _block_applied_callback(fc::variant(block_id)); + }); + } + if(_market_subscriptions.size() == 0) return; - const auto& ops = _db.get_applied_operations(); map< std::pair, vector> > subscribed_markets_ops; for(const auto& op : ops) @@ -1214,6 +1232,13 @@ namespace graphene { namespace app { wdump((result)); return result; } + /** + * Validates a transaction against the current state without broadcast it on the network. + */ + processed_transaction database_api::validate_transaction( const signed_transaction& trx )const + { + return _db.validate_transaction(trx); + } bool database_api::verify_authority( const signed_transaction& trx )const { diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index ae2d5c94..2f546492 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -221,6 +221,15 @@ namespace detail { fc::remove_all(_data_dir / "blockchain/dblock"); } + void set_dbg_init_key( genesis_state_type& genesis, const std::string& init_key ) + { + flat_set< std::string > initial_witness_names; + public_key_type init_pubkey( init_key ); + for( uint64_t i=0; icount("genesis-json") ) { genesis_state_type genesis = fc::json::from_file(_options->at("genesis-json").as()).as(); + bool modified_genesis = false; if( _options->count("genesis-timestamp") ) { genesis.initial_timestamp = fc::time_point_sec( graphene::time::now() ) + genesis.initial_parameters.block_interval + _options->at("genesis-timestamp").as(); genesis.initial_timestamp -= genesis.initial_timestamp.sec_since_epoch() % genesis.initial_parameters.block_interval; + modified_genesis = true; std::cerr << "Used genesis timestamp: " << genesis.initial_timestamp.to_iso_string() << " (PLEASE RECORD THIS)\n"; } + if( _options->count("dbg-init-key") ) + { + std::string init_key = _options->at( "dbg-init-key" ).as(); + FC_ASSERT( genesis.initial_witness_candidates.size() >= genesis.initial_active_witnesses ); + set_dbg_init_key( genesis, init_key ); + modified_genesis = true; + std::cerr << "Set init witness key to " << init_key << "\n"; + } + if( modified_genesis ) + { + std::cerr << "WARNING: GENESIS WAS MODIFIED, YOUR CHAIN ID MAY BE DIFFERENT\n"; + } return genesis; } else @@ -733,6 +756,7 @@ void application::set_program_options(boost::program_options::options_descriptio ("server-pem,p", bpo::value()->implicit_value("server.pem"), "The TLS certificate file for this server") ("server-pem-password,P", bpo::value()->implicit_value(""), "Password for this certificate") ("genesis-json", bpo::value(), "File to read Genesis State from") + ("dbg-init-key", bpo::value(), "Block signing key to use for init witnesses, overrides genesis file") ("api-access", bpo::value(), "JSON file specifying API permissions") ; command_line_options.add(configuration_file_options); @@ -831,6 +855,11 @@ void application::shutdown_plugins() entry.second->plugin_shutdown(); return; } +void application::shutdown() +{ + if( my->_chain_db ) + my->_chain_db->close(); +} void application::initialize_plugins( const boost::program_options::variables_map& options ) { diff --git a/libraries/app/include/graphene/app/api.hpp b/libraries/app/include/graphene/app/api.hpp index ed495010..8a94706f 100644 --- a/libraries/app/include/graphene/app/api.hpp +++ b/libraries/app/include/graphene/app/api.hpp @@ -347,6 +347,11 @@ namespace graphene { namespace app { */ bool verify_account_authority( const string& name_or_id, const flat_set& signers )const; + /** + * Validates a transaction against the current state without broadcast it on the network. + */ + processed_transaction validate_transaction( const signed_transaction& trx )const; + /** * @return the set of blinded balance objects by commitment ID @@ -361,6 +366,8 @@ namespace graphene { namespace app { vector get_required_fees( const vector& ops, asset_id_type id = asset_id_type() )const; void set_subscribe_callback( std::function cb, bool clear_filter ); + void set_pending_transaction_callback( std::function cb ){ _pending_trx_callback = cb; } + void set_block_applied_callback( std::function cb ){ _block_applied_callback = cb; } private: template void subscribe_to_item( const T& i )const @@ -391,10 +398,13 @@ namespace graphene { namespace app { mutable fc::bloom_filter _subscribe_filter; std::function _subscribe_callback; + std::function _pending_trx_callback; + std::function _block_applied_callback; boost::signals2::scoped_connection _change_connection; boost::signals2::scoped_connection _removed_connection; boost::signals2::scoped_connection _applied_block_connection; + boost::signals2::scoped_connection _pending_trx_connection; map< pair, std::function > _market_subscriptions; graphene::chain::database& _db; }; @@ -462,6 +472,8 @@ namespace graphene { namespace app { */ void broadcast_transaction_with_callback( confirmation_callback cb, const signed_transaction& trx); + void broadcast_block( const signed_block& block ); + /** * @brief Not reflected, thus not accessible to API clients. * @@ -595,6 +607,9 @@ FC_API(graphene::app::database_api, (get_blinded_balances) (get_required_fees) (set_subscribe_callback) + (set_pending_transaction_callback) + (set_block_applied_callback) + (validate_transaction) ) FC_API(graphene::app::history_api, (get_account_history) @@ -604,6 +619,7 @@ FC_API(graphene::app::history_api, FC_API(graphene::app::network_broadcast_api, (broadcast_transaction) (broadcast_transaction_with_callback) + (broadcast_block) ) FC_API(graphene::app::network_node_api, (add_node) diff --git a/libraries/chain/block_database.cpp b/libraries/chain/block_database.cpp index c0005971..281dd387 100644 --- a/libraries/chain/block_database.cpp +++ b/libraries/chain/block_database.cpp @@ -237,4 +237,40 @@ optional block_database::last()const } return optional(); } + +optional block_database::last_id()const +{ + try + { + index_entry e; + _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); + + if( _block_num_to_pos.tellp() < sizeof(index_entry) ) + return optional(); + + _block_num_to_pos.seekg( -sizeof(index_entry), _block_num_to_pos.end ); + _block_num_to_pos.read( (char*)&e, sizeof(e) ); + uint64_t pos = _block_num_to_pos.tellg(); + while( e.block_size == 0 && pos > 0 ) + { + pos -= sizeof(index_entry); + _block_num_to_pos.seekg( pos ); + _block_num_to_pos.read( (char*)&e, sizeof(e) ); + } + + if( e.block_size == 0 ) + return optional(); + + return e.block_id; + } + catch (const fc::exception&) + { + } + catch (const std::exception&) + { + } + return optional(); +} + + } } diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp index 6e7f4f52..9014300e 100644 --- a/libraries/chain/db_block.cpp +++ b/libraries/chain/db_block.cpp @@ -17,6 +17,7 @@ */ #include +#include #include #include @@ -95,9 +96,13 @@ bool database::push_block(const signed_block& new_block, uint32_t skip) { idump((new_block.block_num())(new_block.id())); bool result; - with_skip_flags(skip, [&]() + detail::with_skip_flags( *this, skip, [&]() { - result = _push_block(new_block); + detail::without_pending_transactions( *this, std::move(_pending_block.transactions), + [&]() + { + result = _push_block(new_block); + }); }); return result; } @@ -166,11 +171,6 @@ bool database::_push_block(const signed_block& new_block) } } - // If there is a pending block session, then the database state is dirty with pending transactions. - // Drop the pending session to reset the database to a clean head block state. - // TODO: Preserve pending transactions, and re-apply any which weren't included in the new block. - clear_pending(); - try { auto session = _undo_db.start_undo_session(); apply_block(new_block, skip); @@ -197,7 +197,7 @@ bool database::_push_block(const signed_block& new_block) processed_transaction database::push_transaction( const signed_transaction& trx, uint32_t skip ) { try { processed_transaction result; - with_skip_flags( skip, [&]() + detail::with_skip_flags( *this, skip, [&]() { result = _push_transaction( trx ); } ); @@ -220,9 +220,18 @@ processed_transaction database::_push_transaction( const signed_transaction& trx notify_changed_objects(); // The transaction applied successfully. Merge its changes into the pending block session. session.merge(); + + // notify anyone listening to pending transactions + on_pending_transaction( trx ); return processed_trx; } +processed_transaction database::validate_transaction( const signed_transaction& trx ) +{ + auto session = _undo_db.start_undo_session(); + return _apply_transaction( trx ); +} + processed_transaction database::push_proposal(const proposal_object& proposal) { transaction_evaluation_state eval_state(this); @@ -250,7 +259,7 @@ signed_block database::generate_block( ) { signed_block result; - with_skip_flags( skip, [&]() + detail::with_skip_flags( *this, skip, [&]() { result = _generate_block( when, witness_id, block_signing_private_key, true ); } ); @@ -391,7 +400,7 @@ void database::apply_block( const signed_block& next_block, uint32_t skip ) skip = ~0;// WE CAN SKIP ALMOST EVERYTHING } - with_skip_flags( skip, [&]() + detail::with_skip_flags( *this, skip, [&]() { _apply_block( next_block ); } ); @@ -480,7 +489,7 @@ void database::notify_changed_objects() processed_transaction database::apply_transaction(const signed_transaction& trx, uint32_t skip) { processed_transaction result; - with_skip_flags(skip, [&]() + detail::with_skip_flags( *this, skip, [&]() { result = _apply_transaction(trx); }); diff --git a/libraries/chain/db_init.cpp b/libraries/chain/db_init.cpp index f3361bf1..55122d12 100644 --- a/libraries/chain/db_init.cpp +++ b/libraries/chain/db_init.cpp @@ -331,13 +331,13 @@ void database::init_genesis(const genesis_state_type& genesis_state) // Create more special assets while( true ) { + uint64_t id = get_index().get_next_id().instance(); + if( id >= genesis_state.immutable_parameters.num_special_assets ) + break; const asset_dynamic_data_object& dyn_asset = create([&](asset_dynamic_data_object& a) { a.current_supply = 0; }); - uint64_t id = get_index().get_next_id().instance(); - if( id >= genesis_state.immutable_parameters.num_special_assets ) - break; const asset_object& asset_obj = create( [&]( asset_object& a ) { a.symbol = "SPECIAL" + std::to_string( id ); a.options.max_supply = 0; diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp index 4a5cf899..c33b3ad1 100644 --- a/libraries/chain/db_management.cpp +++ b/libraries/chain/db_management.cpp @@ -58,12 +58,32 @@ void database::reindex(fc::path data_dir, const genesis_state_type& initial_allo for( uint32_t i = 1; i <= last_block_num; ++i ) { if( i % 2000 == 0 ) std::cerr << " " << double(i*100)/last_block_num << "% "< block = _block_id_to_block.fetch_by_number(i); + if( !block.valid() ) + { + wlog( "Reindexing terminated due to gap: Block ${i} does not exist!", ("i", i) ); + uint32_t dropped_count = 0; + while( true ) + { + fc::optional< block_id_type > last_id = _block_id_to_block.last_id(); + // this can trigger if we attempt to e.g. read a file that has block #2 but no block #1 + if( !last_id.valid() ) + break; + // we've caught up to the gap + if( block_header::num_from_id( *last_id ) <= i ) + break; + _block_id_to_block.remove( *last_id ); + dropped_count++; + } + wlog( "Dropped ${n} blocks from after the gap", ("n", dropped_count) ); + break; + } + apply_block(*block, skip_witness_signature | + skip_transaction_signatures | + skip_transaction_dupe_check | + skip_tapos_check | + skip_witness_schedule_check | + skip_authority_check); } _undo_db.enable(); auto end = fc::time_point::now(); @@ -101,9 +121,12 @@ void database::open( { _fork_db.start_block( *last_block ); idump((last_block->id())(last_block->block_num())); + if( last_block->id() != head_block_id() ) + { + FC_ASSERT( head_block_num() == 0, "last block ID does not match current chain state" ); + } } idump((head_block_id())(head_block_num())); - FC_ASSERT( !last_block || last_block->id() == head_block_id() ); } FC_CAPTURE_AND_RETHROW( (data_dir) ) } diff --git a/libraries/chain/db_update.cpp b/libraries/chain/db_update.cpp index 11e5c0e0..debb05a0 100644 --- a/libraries/chain/db_update.cpp +++ b/libraries/chain/db_update.cpp @@ -17,6 +17,7 @@ */ #include +#include #include #include @@ -100,10 +101,6 @@ void database::update_pending_block(const signed_block& next_block, uint8_t curr { _pending_block.timestamp = next_block.timestamp + current_block_interval; _pending_block.previous = next_block.id(); - auto old_pending_trx = std::move(_pending_block.transactions); - _pending_block.transactions.clear(); - for( auto old_trx : old_pending_trx ) - push_transaction( old_trx ); } void database::clear_expired_transactions() @@ -112,9 +109,7 @@ void database::clear_expired_transactions() //Transactions must have expired by at least two forking windows in order to be removed. auto& transaction_idx = static_cast(get_mutable_index(implementation_ids, impl_transaction_object_type)); const auto& dedupe_index = transaction_idx.indices().get(); - const auto& global_parameters = get_global_properties().parameters; - while( !dedupe_index.empty() - && head_block_time() - dedupe_index.rbegin()->trx.expiration >= fc::seconds(global_parameters.maximum_expiration) ) + while( (!dedupe_index.empty()) && (head_block_time() > dedupe_index.rbegin()->trx.expiration) ) transaction_idx.remove(*dedupe_index.rbegin()); } @@ -142,7 +137,7 @@ void database::clear_expired_proposals() void database::clear_expired_orders() { - with_skip_flags( + detail::with_skip_flags( *this, get_node_properties().skip_flags | skip_authority_check, [&](){ transaction_evaluation_state cancel_context(this); @@ -158,7 +153,6 @@ void database::clear_expired_orders() } }); - //Process expired force settlement orders auto& settlement_index = get_index_type().indices().get(); if( !settlement_index.empty() ) diff --git a/libraries/chain/get_config.cpp b/libraries/chain/get_config.cpp index 9f10ce78..1d81ecfb 100644 --- a/libraries/chain/get_config.cpp +++ b/libraries/chain/get_config.cpp @@ -45,7 +45,6 @@ fc::variant_object get_config() result[ "GRAPHENE_DEFAULT_MAX_TIME_UNTIL_EXPIRATION" ] = GRAPHENE_DEFAULT_MAX_TIME_UNTIL_EXPIRATION; result[ "GRAPHENE_DEFAULT_MAINTENANCE_INTERVAL" ] = GRAPHENE_DEFAULT_MAINTENANCE_INTERVAL; result[ "GRAPHENE_DEFAULT_MAINTENANCE_SKIP_SLOTS" ] = GRAPHENE_DEFAULT_MAINTENANCE_SKIP_SLOTS; - result[ "GRAPHENE_DEFAULT_MAX_EXPIRATION_SEC" ] = GRAPHENE_DEFAULT_MAX_EXPIRATION_SEC; result[ "GRAPHENE_MIN_UNDO_HISTORY" ] = GRAPHENE_MIN_UNDO_HISTORY; result[ "GRAPHENE_MAX_UNDO_HISTORY" ] = GRAPHENE_MAX_UNDO_HISTORY; result[ "GRAPHENE_MIN_BLOCK_SIZE_LIMIT" ] = GRAPHENE_MIN_BLOCK_SIZE_LIMIT; diff --git a/libraries/chain/include/graphene/chain/block_database.hpp b/libraries/chain/include/graphene/chain/block_database.hpp index 1e8a97a6..816df798 100644 --- a/libraries/chain/include/graphene/chain/block_database.hpp +++ b/libraries/chain/include/graphene/chain/block_database.hpp @@ -36,6 +36,7 @@ namespace graphene { namespace chain { optional fetch_optional( const block_id_type& id )const; optional fetch_by_number( uint32_t block_num )const; optional last()const; + optional last_id()const; private: mutable std::fstream _blocks; mutable std::fstream _block_num_to_pos; diff --git a/libraries/chain/include/graphene/chain/config.hpp b/libraries/chain/include/graphene/chain/config.hpp index 7256fbb2..3ad559ea 100644 --- a/libraries/chain/include/graphene/chain/config.hpp +++ b/libraries/chain/include/graphene/chain/config.hpp @@ -45,7 +45,6 @@ #define GRAPHENE_DEFAULT_MAX_TIME_UNTIL_EXPIRATION (60*60*24) // seconds, aka: 1 day #define GRAPHENE_DEFAULT_MAINTENANCE_INTERVAL (60*60*24) // seconds, aka: 1 day #define GRAPHENE_DEFAULT_MAINTENANCE_SKIP_SLOTS 3 // number of slots to skip for maintenance interval -#define GRAPHENE_DEFAULT_MAX_EXPIRATION_SEC (60*60) // 1 hour #define GRAPHENE_MIN_UNDO_HISTORY 10 #define GRAPHENE_MAX_UNDO_HISTORY 1000 diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp index e5687fd1..4445654a 100644 --- a/libraries/chain/include/graphene/chain/database.hpp +++ b/libraries/chain/include/graphene/chain/database.hpp @@ -40,31 +40,6 @@ namespace graphene { namespace chain { using graphene::db::abstract_object; using graphene::db::object; - namespace detail - { - /** - * Class used to help the with_skip_flags implementation. - * It must be defined in this header because it must be - * available to the with_skip_flags implementation, - * which is a template and therefore must also be defined - * in this header. - */ - struct skip_flags_restorer - { - skip_flags_restorer( node_property_object& npo, uint32_t old_skip_flags ) - : _npo( npo ), _old_skip_flags( old_skip_flags ) - {} - - ~skip_flags_restorer() - { - _npo.skip_flags = _old_skip_flags; - } - - node_property_object& _npo; - uint32_t _old_skip_flags; - }; - } - /** * @class database * @brief tracks the blockchain state in an extensible manner @@ -197,6 +172,12 @@ namespace graphene { namespace chain { */ fc::signal applied_block; + /** + * This signal is emitted any time a new transaction is added to the pending + * block state. + */ + fc::signal on_pending_transaction; + /** * Emitted After a block has been applied and committed. The callback * should not yield and should execute quickly. @@ -265,23 +246,7 @@ namespace graphene { namespace chain { node_property_object& node_properties(); - /** - * Set the skip_flags to the given value, call callback, - * then reset skip_flags to their previous value after - * callback is done. - */ - template< typename Lambda > - void with_skip_flags( - uint32_t skip_flags, - Lambda callback ) - { - node_property_object& npo = node_properties(); - detail::skip_flags_restorer restorer( npo, npo.skip_flags ); - npo.skip_flags = skip_flags; - callback(); - return; - } - + uint32_t last_non_undoable_block_num() const; //////////////////// db_init.cpp //////////////////// @@ -406,7 +371,13 @@ namespace graphene { namespace chain { asset calculate_market_fee(const asset_object& recv_asset, const asset& trade_amount); asset pay_market_fees( const asset_object& recv_asset, const asset& receives ); + ///@} + /** + * This method validates transactions without adding it to the pending state. + * @return true if the transaction would validate + */ + processed_transaction validate_transaction( const signed_transaction& trx ); /** * @} @@ -431,6 +402,7 @@ namespace graphene { namespace chain { processed_transaction _apply_transaction( const signed_transaction& trx ); operation_result apply_operation( transaction_evaluation_state& eval_state, const operation& op ); + ///Steps involved in applying a new block ///@{ diff --git a/libraries/chain/include/graphene/chain/db_with.hpp b/libraries/chain/include/graphene/chain/db_with.hpp new file mode 100644 index 00000000..09781f0f --- /dev/null +++ b/libraries/chain/include/graphene/chain/db_with.hpp @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2015, Cryptonomex, Inc. + * All rights reserved. + * + * This source code is provided for evaluation in private test networks only, until September 8, 2015. After this date, this license expires and + * the code may not be used, modified or distributed for any purpose. Redistribution and use in source and binary forms, with or without modification, + * are permitted until September 8, 2015, provided that the following conditions are met: + * + * 1. The code and/or derivative works are used only for private test networks consisting of no more than 10 P2P nodes. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#pragma once + +#include + +/* + * This file provides with() functions which modify the database + * temporarily, then restore it. These functions are mostly internal + * implementation detail of the database. + * + * Essentially, we want to be able to use "finally" to restore the + * database regardless of whether an exception is thrown or not, but there + * is no "finally" in C++. Instead, C++ requires us to create a struct + * and put the finally block in a destructor. Aagh! + */ + +namespace graphene { namespace chain { namespace detail { +/** + * Class used to help the with_skip_flags implementation. + * It must be defined in this header because it must be + * available to the with_skip_flags implementation, + * which is a template and therefore must also be defined + * in this header. + */ +struct skip_flags_restorer +{ + skip_flags_restorer( node_property_object& npo, uint32_t old_skip_flags ) + : _npo( npo ), _old_skip_flags( old_skip_flags ) + {} + + ~skip_flags_restorer() + { + _npo.skip_flags = _old_skip_flags; + } + + node_property_object& _npo; + uint32_t _old_skip_flags; +}; + +/** + * Class used to help the without_pending_transactions + * implementation. + */ +struct pending_transactions_restorer +{ + pending_transactions_restorer( database& db, std::vector&& pending_transactions ) + : _db(db), _pending_transactions( std::move(pending_transactions) ) + { + _db.clear_pending(); + } + + ~pending_transactions_restorer() + { + for( const processed_transaction& tx : _pending_transactions ) + { + try + { + // since push_transaction() takes a signed_transaction, + // the operation_results field will be ignored. + _db.push_transaction( tx ); + } + catch( const fc::exception& e ) + { + wlog( "Pending transaction became invalid after switching to block ${b}", ("b", _db.head_block_id()) ); + wlog( "The invalid pending transaction is ${t}", ("t", tx) ); + wlog( "The invalid pending transaction caused exception ${e}", ("e", e) ); + } + } + } + + database& _db; + std::vector< processed_transaction > _pending_transactions; +}; + +/** + * Set the skip_flags to the given value, call callback, + * then reset skip_flags to their previous value after + * callback is done. + */ +template< typename Lambda > +void with_skip_flags( + database& db, + uint32_t skip_flags, + Lambda callback ) +{ + node_property_object& npo = db.node_properties(); + skip_flags_restorer restorer( npo, npo.skip_flags ); + npo.skip_flags = skip_flags; + callback(); + return; +} + +/** + * Empty pending_transactions, call callback, + * then reset pending_transactions after callback is done. + * + * Pending transactions which no longer validate will be culled. + */ +template< typename Lambda > +void without_pending_transactions( + database& db, + std::vector&& pending_transactions, + Lambda callback ) +{ + pending_transactions_restorer restorer( db, std::move(pending_transactions) ); + callback(); + return; +} + +} } } // graphene::chain::detail diff --git a/libraries/chain/include/graphene/chain/protocol/chain_parameters.hpp b/libraries/chain/include/graphene/chain/protocol/chain_parameters.hpp index ac91d777..1f0e8d76 100644 --- a/libraries/chain/include/graphene/chain/protocol/chain_parameters.hpp +++ b/libraries/chain/include/graphene/chain/protocol/chain_parameters.hpp @@ -41,7 +41,7 @@ namespace graphene { namespace chain { uint32_t committee_proposal_review_period = GRAPHENE_DEFAULT_COMMITTEE_PROPOSAL_REVIEW_PERIOD_SEC; ///< minimum time in seconds that a proposed transaction requiring committee authority may not be signed, prior to expiration uint32_t maximum_transaction_size = GRAPHENE_DEFAULT_MAX_TRANSACTION_SIZE; ///< maximum allowable size in bytes for a transaction uint32_t maximum_block_size = GRAPHENE_DEFAULT_MAX_BLOCK_SIZE; ///< maximum allowable size in bytes for a block - uint32_t maximum_expiration = GRAPHENE_DEFAULT_MAX_EXPIRATION_SEC; ///< maximum number of seconds in the future a transaction may expire + uint32_t maximum_expiration = 0; ///< ignored, but included to ensure we don't hardfork; see #308 uint32_t maximum_time_until_expiration = GRAPHENE_DEFAULT_MAX_TIME_UNTIL_EXPIRATION; ///< maximum lifetime in seconds for transactions to be valid, before expiring uint32_t maximum_proposal_lifetime = GRAPHENE_DEFAULT_MAX_PROPOSAL_LIFETIME_SEC; ///< maximum lifetime in seconds for proposed transactions to be kept, before expiring uint8_t maximum_asset_whitelist_authorities = GRAPHENE_DEFAULT_MAX_ASSET_WHITELIST_AUTHORITIES; ///< maximum number of accounts which an asset may list as authorities for its whitelist OR blacklist diff --git a/libraries/net/node.cpp b/libraries/net/node.cpp index b2ef745d..e3f75360 100644 --- a/libraries/net/node.cpp +++ b/libraries/net/node.cpp @@ -1198,9 +1198,9 @@ namespace graphene { namespace net { namespace detail { wdump((inventory_to_advertise)); for (const item_id& item_to_advertise : inventory_to_advertise) { - if (peer->inventory_advertised_to_peer.find(item_to_advertise) == peer->inventory_advertised_to_peer.end() ) + if (peer->inventory_advertised_to_peer.find(item_to_advertise) != peer->inventory_advertised_to_peer.end() ) wdump((*peer->inventory_advertised_to_peer.find(item_to_advertise))); - if (peer->inventory_peer_advertised_to_us.find(item_to_advertise) == peer->inventory_peer_advertised_to_us.end() ) + if (peer->inventory_peer_advertised_to_us.find(item_to_advertise) != peer->inventory_peer_advertised_to_us.end() ) wdump((*peer->inventory_peer_advertised_to_us.find(item_to_advertise))); if (peer->inventory_advertised_to_peer.find(item_to_advertise) == peer->inventory_advertised_to_peer.end() && diff --git a/libraries/plugins/witness/include/graphene/witness/witness.hpp b/libraries/plugins/witness/include/graphene/witness/witness.hpp index c82b83d8..df033094 100644 --- a/libraries/plugins/witness/include/graphene/witness/witness.hpp +++ b/libraries/plugins/witness/include/graphene/witness/witness.hpp @@ -75,6 +75,7 @@ private: bool _production_enabled = false; bool _consecutive_production_enabled = false; uint32_t _required_witness_participation = 33 * GRAPHENE_1_PERCENT; + uint32_t _production_skip_flags = graphene::chain::database::skip_nothing; std::map _private_keys; std::set _witnesses; diff --git a/libraries/plugins/witness/witness.cpp b/libraries/plugins/witness/witness.cpp index 24eb0309..db5154ec 100644 --- a/libraries/plugins/witness/witness.cpp +++ b/libraries/plugins/witness/witness.cpp @@ -123,8 +123,12 @@ void witness_plugin::plugin_startup() { ilog("Launching block production for ${n} witnesses.", ("n", _witnesses.size())); app().set_block_production(true); - if( _production_enabled && (d.head_block_num() == 0) ) - new_chain_banner(d); + if( _production_enabled ) + { + if( d.head_block_num() == 0 ) + new_chain_banner(d); + _production_skip_flags |= graphene::chain::database::skip_undo_history_check; + } schedule_production_loop(); } else elog("No witnesses configured! Please add witness IDs and private keys to configuration."); @@ -278,7 +282,7 @@ block_production_condition::block_production_condition_enum witness_plugin::mayb scheduled_time, scheduled_witness, private_key_itr->second, - graphene::chain::database::skip_nothing + _production_skip_flags ); capture("n", block.block_num())("t", block.timestamp)("c", now); p2p_node().broadcast(net::block_message(block)); diff --git a/libraries/wallet/wallet.cpp b/libraries/wallet/wallet.cpp index 09d7ee29..7d5cef4b 100644 --- a/libraries/wallet/wallet.cpp +++ b/libraries/wallet/wallet.cpp @@ -375,6 +375,12 @@ public: ("chain_id", _chain_id) ); } init_prototype_ops(); + + _remote_db->set_block_applied_callback( [this](const variant& block_id ) + { + on_block_applied( block_id ); + } ); + _wallet.chain_id = _chain_id; _wallet.ws_server = initial_data.ws_server; _wallet.ws_user = initial_data.ws_user; @@ -408,6 +414,11 @@ public: } } + void on_block_applied( const variant& block_id ) + { + fc::async([this]{resync();}, "Resync after block"); + } + bool copy_wallet_file( string destination_filename ) { fc::path src_path = get_wallet_filename(); @@ -648,6 +659,51 @@ public: FC_THROW( "Wallet chain ID does not match", ("wallet.chain_id", _wallet.chain_id) ("chain_id", _chain_id) ); + + size_t account_pagination = 100; + vector< account_id_type > account_ids_to_send; + size_t n = _wallet.my_accounts.size(); + account_ids_to_send.reserve( std::min( account_pagination, n ) ); + auto it = _wallet.my_accounts.begin(); + + for( size_t start=0; start start ); + account_ids_to_send.clear(); + std::vector< account_object > old_accounts; + for( size_t i=start; i > accounts = _remote_db->get_accounts(account_ids_to_send); + // server response should be same length as request + FC_ASSERT( accounts.size() == account_ids_to_send.size() ); + size_t i = 0; + for( const optional< account_object >& acct : accounts ) + { + account_object& old_acct = old_accounts[i]; + if( !acct.valid() ) + { + elog( "Could not find account ${id} : \"${name}\" does not exist on the chain!", ("id", old_acct.id)("name", old_acct.name) ); + i++; + continue; + } + // this check makes sure the server didn't send results + // in a different order, or accounts we didn't request + FC_ASSERT( acct->id == old_acct.id ); + if( fc::json::to_string(*acct) != fc::json::to_string(old_acct) ) + { + wlog( "Account ${id} : \"${name}\" updated on chain", ("id", acct->id)("name", acct->name) ); + } + _wallet.update_account( *acct ); + i++; + } + } + return true; } void save_wallet_file(string wallet_filename = "") diff --git a/programs/witness_node/main.cpp b/programs/witness_node/main.cpp index 74126d39..97c7a4a9 100644 --- a/programs/witness_node/main.cpp +++ b/programs/witness_node/main.cpp @@ -53,8 +53,8 @@ void write_default_logging_config_to_stream(std::ostream& out); fc::optional load_logging_config_from_ini_file(const fc::path& config_ini_filename); int main(int argc, char** argv) { + app::application* node = new app::application(); try { - app::application node; bpo::options_description app_options("Graphene Witness Node"); bpo::options_description cfg_options("Graphene Witness Node"); app_options.add_options() @@ -64,14 +64,14 @@ int main(int argc, char** argv) { bpo::variables_map options; - auto witness_plug = node.register_plugin(); - auto history_plug = node.register_plugin(); - auto market_history_plug = node.register_plugin(); + auto witness_plug = node->register_plugin(); + auto history_plug = node->register_plugin(); + auto market_history_plug = node->register_plugin(); try { bpo::options_description cli, cfg; - node.set_program_options(cli, cfg); + node->set_program_options(cli, cfg); app_options.add(cli); cfg_options.add(cfg); bpo::store(bpo::parse_command_line(argc, argv, app_options), options); @@ -151,26 +151,31 @@ int main(int argc, char** argv) { } bpo::notify(options); - node.initialize(data_dir, options); - node.initialize_plugins( options ); + node->initialize(data_dir, options); + node->initialize_plugins( options ); - node.startup(); - node.startup_plugins(); + node->startup(); + node->startup_plugins(); fc::promise::ptr exit_promise = new fc::promise("UNIX Signal Handler"); fc::set_signal_handler([&exit_promise](int signal) { + elog( "Caught ^C attempting to exit cleanly" ); exit_promise->set_value(signal); }, SIGINT); - ilog("Started witness node on a chain with ${h} blocks.", ("h", node.chain_database()->head_block_num())); - ilog("Chain ID is ${id}", ("id", node.chain_database()->get_chain_id()) ); + ilog("Started witness node on a chain with ${h} blocks.", ("h", node->chain_database()->head_block_num())); + ilog("Chain ID is ${id}", ("id", node->chain_database()->get_chain_id()) ); int signal = exit_promise->wait(); ilog("Exiting from signal ${n}", ("n", signal)); - node.shutdown_plugins(); + node->shutdown_plugins(); + node->shutdown(); + delete node; return 0; } catch( const fc::exception& e ) { elog("Exiting with error:\n${e}", ("e", e.to_detail_string())); + node->shutdown(); + delete node; return 1; } } diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp index 2504e7ce..4a904390 100644 --- a/tests/tests/block_tests.cpp +++ b/tests/tests/block_tests.cpp @@ -127,12 +127,14 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) // TODO: Don't generate this here auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); + signed_block b200; { database db; db.open(data_dir.path(), make_genesis ); b = db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); - for( uint32_t i = 1; i < 200; ++i ) + // n.b. we generate GRAPHENE_MIN_UNDO_HISTORY+1 extra blocks which will be discarded on save + for( uint32_t i = 1; i < 200+GRAPHENE_MIN_UNDO_HISTORY+1; ++i ) { BOOST_CHECK( db.head_block_id() == b.id() ); witness_id_type prev_witness = b.witness; @@ -140,6 +142,8 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) BOOST_CHECK( cur_witness != prev_witness ); b = db.generate_block(db.get_slot_time(1), cur_witness, init_account_priv_key, database::skip_nothing); BOOST_CHECK( b.witness == cur_witness ); + if( i == 199 ) + b200 = b; } db.close(); } @@ -147,6 +151,7 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) database db; db.open(data_dir.path(), []{return genesis_state_type();}); BOOST_CHECK_EQUAL( db.head_block_num(), 200 ); + b = b200; for( uint32_t i = 0; i < 200; ++i ) { BOOST_CHECK( db.head_block_id() == b.id() ); @@ -1018,4 +1023,157 @@ BOOST_FIXTURE_TEST_CASE( rsf_missed_blocks, database_fixture ) FC_LOG_AND_RETHROW() } +BOOST_FIXTURE_TEST_CASE( transaction_invalidated_in_cache, database_fixture ) +{ + try + { + ACTORS( (alice)(bob) ); + + auto generate_block = [&]( database& d ) -> signed_block + { + return d.generate_block(d.get_slot_time(1), d.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + }; + + wdump( (db.fetch_block_by_number(1)) ); + wdump( (db.fetch_block_by_id( db.head_block_id() ) ) ); + + signed_block b1 = generate_block(db); + wdump( (db.fetch_block_by_number(1)) ); + wdump( (db.fetch_block_by_id( db.head_block_id() ) ) ); + + fc::temp_directory data_dir2( graphene::utilities::temp_directory_path() ); + + database db2; + db2.open(data_dir2.path(), make_genesis); + BOOST_CHECK( db.get_chain_id() == db2.get_chain_id() ); + + while( db2.head_block_num() < db.head_block_num() ) + { + wdump( (db.head_block_num()) (db2.head_block_num()) ); + optional< signed_block > b = db.fetch_block_by_number( db2.head_block_num()+1 ); + db2.push_block(*b, database::skip_witness_signature); + } + wlog("caught up db2 to db"); + BOOST_CHECK( db2.get( alice_id ).name == "alice" ); + BOOST_CHECK( db2.get( bob_id ).name == "bob" ); + + db2.push_block(generate_block(db)); + transfer( account_id_type(), alice_id, asset( 1000 ) ); + transfer( account_id_type(), bob_id, asset( 1000 ) ); + db2.push_block(generate_block(db)); + + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 1000); + BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 1000); + BOOST_CHECK_EQUAL(db2.get_balance(alice_id, asset_id_type()).amount.value, 1000); + BOOST_CHECK_EQUAL(db2.get_balance( bob_id, asset_id_type()).amount.value, 1000); + + auto generate_and_send = [&]( int n ) + { + for( int i=0; i signed_transaction + { + signed_transaction tx; + transfer_operation xfer_op; + xfer_op.from = from; + xfer_op.to = to; + xfer_op.amount = asset( amount, asset_id_type() ); + xfer_op.fee = asset( 0, asset_id_type() ); + tx.operations.push_back( xfer_op ); + tx.set_expiration( db.head_block_time() + blocks_to_expire * db.get_global_properties().parameters.block_interval ); + if( from == alice_id ) + sign( tx, alice_private_key ); + else + sign( tx, bob_private_key ); + return tx; + }; + + signed_transaction tx = generate_xfer_tx( alice_id, bob_id, 1000, 2 ); + tx.set_expiration( db.head_block_time() + 2 * db.get_global_properties().parameters.block_interval ); + tx.signatures.clear(); + sign( tx, alice_private_key ); + // put the tx in db tx cache + PUSH_TX( db, tx ); + + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 0); + BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 2000); + + // generate some blocks with db2, make tx expire in db's cache + generate_and_send(3); + + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 1000); + BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 1000); + + // generate a block with db and ensure we don't somehow apply it + PUSH_BLOCK(db2, generate_block(db)); + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 1000); + BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 1000); + + // now the tricky part... + // (A) Bob sends 1000 to Alice + // (B) Alice sends 2000 to Bob + // (C) Alice sends 500 to Bob + // + // We push AB, then receive a block containing C. + // we need to apply the block, then invalidate B in the cache. + // AB results in Alice having 0, Bob having 2000. + // C results in Alice having 500, Bob having 1500. + // + // This needs to occur while switching to a fork. + // + + signed_transaction tx_a = generate_xfer_tx( bob_id, alice_id, 1000, 3 ); + signed_transaction tx_b = generate_xfer_tx( alice_id, bob_id, 2000 ); + signed_transaction tx_c = generate_xfer_tx( alice_id, bob_id, 500 ); + + generate_block( db ); + + PUSH_TX( db, tx_a ); + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 2000); + BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 0); + + PUSH_TX( db, tx_b ); + PUSH_TX( db2, tx_c ); + + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 0); + BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 2000); + + BOOST_CHECK_EQUAL(db2.get_balance(alice_id, asset_id_type()).amount.value, 500); + BOOST_CHECK_EQUAL(db2.get_balance( bob_id, asset_id_type()).amount.value, 1500); + + // generate enough blocks on db2 to cause db to switch forks + generate_and_send(2); + + // db should invalidate B, but still be applying A, so the states don't agree + + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 1500); + BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 500); + + BOOST_CHECK_EQUAL(db2.get_balance(alice_id, asset_id_type()).amount.value, 500); + BOOST_CHECK_EQUAL(db2.get_balance( bob_id, asset_id_type()).amount.value, 1500); + + // This will cause A to expire in db + generate_and_send(1); + + BOOST_CHECK_EQUAL(db.get_balance(alice_id, asset_id_type()).amount.value, 500); + BOOST_CHECK_EQUAL(db.get_balance( bob_id, asset_id_type()).amount.value, 1500); + + BOOST_CHECK_EQUAL(db2.get_balance(alice_id, asset_id_type()).amount.value, 500); + BOOST_CHECK_EQUAL(db2.get_balance( bob_id, asset_id_type()).amount.value, 1500); + + // Make sure we can generate and accept a plain old empty block on top of all this! + generate_and_send(1); + } + catch (fc::exception& e) + { + edump((e.to_detail_string())); + throw; + } +} + BOOST_AUTO_TEST_SUITE_END()