diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 19bbc9e0..8355d795 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,6 @@ +include: + - template: Code-Quality.gitlab-ci.yml + stages: - build - test @@ -24,5 +27,28 @@ test: script: - ./tests/betting_test - ./tests/chain_test + - ./tests/cli_test tags: - - builder \ No newline at end of file + - builder + +code_quality: + stage: test + image: docker:stable + variables: + DOCKER_DRIVER: overlay2 + allow_failure: true + services: + - docker:stable-dind + script: + - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + - docker run + --env SOURCE_CODE="$PWD" + --volume "$PWD":/code + --volume /var/run/docker.sock:/var/run/docker.sock + "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code + artifacts: + paths: [gl-code-quality-report.json] + expire_in: 1 week + except: + variables: + - $CODE_QUALITY_DISABLED diff --git a/.sonarcloud.properties b/.sonarcloud.properties new file mode 100644 index 00000000..e69de29b diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index 318ad821..73861eb8 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -160,7 +160,10 @@ namespace graphene { namespace app { { auto block_num = b.block_num(); auto& callback = _callbacks.find(id)->second; - fc::async( [capture_this,this,id,block_num,trx_num,trx,callback](){ callback( fc::variant(transaction_confirmation{ id, block_num, trx_num, trx}) ); } ); + fc::async( [capture_this,this,id,block_num,trx_num,trx,callback]() { + callback( fc::variant( transaction_confirmation{ id, block_num, trx_num, trx }, + GRAPHENE_MAX_NESTED_OBJECTS ) ); + } ); } } } @@ -171,7 +174,8 @@ namespace graphene { namespace app { trx.validate(); _app.chain_database()->check_tansaction_for_duplicated_operations(trx); _app.chain_database()->push_transaction(trx); - _app.p2p_node()->broadcast_transaction(trx); + if( _app.p2p_node() != nullptr ) + _app.p2p_node()->broadcast_transaction(trx); } fc::variant network_broadcast_api::broadcast_transaction_synchronous(const signed_transaction& trx) @@ -189,7 +193,8 @@ namespace graphene { namespace app { void network_broadcast_api::broadcast_block( const signed_block& b ) { _app.chain_database()->push_block(b); - _app.p2p_node()->broadcast( net::block_message( b )); + if( _app.p2p_node() != nullptr ) + _app.p2p_node()->broadcast( net::block_message( b )); } void network_broadcast_api::broadcast_transaction_with_callback(confirmation_callback cb, const signed_transaction& trx) @@ -197,7 +202,8 @@ namespace graphene { namespace app { trx.validate(); _callbacks[trx.id()] = cb; _app.chain_database()->push_transaction(trx); - _app.p2p_node()->broadcast_transaction(trx); + if( _app.p2p_node() != nullptr ) + _app.p2p_node()->broadcast_transaction(trx); } network_node_api::network_node_api( application& a ) : _app( a ) @@ -212,7 +218,7 @@ namespace graphene { namespace app { if (_on_pending_transaction) { - _on_pending_transaction(fc::variant(transaction)); + _on_pending_transaction(fc::variant(transaction, GRAPHENE_MAX_NESTED_OBJECTS)); } }); diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index 4f7ddef9..0f0c0690 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -142,7 +142,7 @@ namespace detail { if( _options->count("seed-nodes") ) { auto seeds_str = _options->at("seed-nodes").as(); - auto seeds = fc::json::from_string(seeds_str).as>(); + auto seeds = fc::json::from_string(seeds_str).as>(2); for( const string& endpoint_string : seeds ) { try { @@ -226,7 +226,7 @@ namespace detail { void new_connection( const fc::http::websocket_connection_ptr& c ) { - auto wsc = std::make_shared(c); + auto wsc = std::make_shared(c, GRAPHENE_MAX_NESTED_OBJECTS); auto login = std::make_shared( std::ref(*_self) ); login->enable_api("database_api"); @@ -292,7 +292,7 @@ namespace detail { _websocket_tls_server->start_accept(); } FC_CAPTURE_AND_RETHROW() } - application_impl(application* self) + explicit application_impl(application* self) : _self(self), _chain_db(std::make_shared()) { @@ -300,7 +300,6 @@ namespace detail { ~application_impl() { - fc::remove_all(_data_dir / "blockchain/dblock"); } void set_dbg_init_key( genesis_state_type& genesis, const std::string& init_key ) @@ -309,21 +308,19 @@ namespace detail { public_key_type init_pubkey( init_key ); for( uint64_t i=0; icount("genesis-json") ) { std::string genesis_str; fc::read_file_contents( _options->at("genesis-json").as(), genesis_str ); - genesis_state_type genesis = fc::json::from_string( genesis_str ).as(); + genesis_state_type genesis = fc::json::from_string( genesis_str ).as( 20 ); bool modified_genesis = false; if( _options->count("genesis-timestamp") ) { @@ -356,7 +353,7 @@ namespace detail { graphene::egenesis::compute_egenesis_json( egenesis_json ); FC_ASSERT( egenesis_json != "" ); FC_ASSERT( graphene::egenesis::get_egenesis_json_hash() == fc::sha256::hash( egenesis_json ) ); - auto genesis = fc::json::from_string( egenesis_json ).as(); + auto genesis = fc::json::from_string( egenesis_json ).as( 20 ); genesis.initial_chain_id = fc::sha256::hash( egenesis_json ); return genesis; } @@ -372,7 +369,7 @@ namespace detail { loaded_checkpoints.reserve( cps.size() ); for( auto cp : cps ) { - auto item = fc::json::from_string(cp).as >(); + auto item = fc::json::from_string(cp).as >( 2 ); loaded_checkpoints[item.first] = item.second; } } @@ -381,64 +378,17 @@ namespace detail { bool replay = false; std::string replay_reason = "reason not provided"; - // never replay if data dir is empty - if( fc::exists( _data_dir ) && fc::directory_iterator( _data_dir ) != fc::directory_iterator() ) - { - if( _options->count("replay-blockchain") ) - { - replay = true; - replay_reason = "replay-blockchain argument specified"; - } - else if( !clean ) - { - replay = true; - replay_reason = "unclean shutdown detected"; - } - else if( !fc::exists( _data_dir / "db_version" ) ) - { - replay = true; - replay_reason = "db_version file not found"; - } - else - { - std::string version_string; - fc::read_file_contents( _data_dir / "db_version", version_string ); + if( _options->count("replay-blockchain") ) + _chain_db->wipe( _data_dir / "blockchain", false ); - if( version_string != GRAPHENE_CURRENT_DB_VERSION ) - { - replay = true; - replay_reason = "db_version file content mismatch"; - } - } + try + { + _chain_db->open( _data_dir / "blockchain", initial_state, GRAPHENE_CURRENT_DB_VERSION ); } - - if( !replay ) + catch( const fc::exception& e ) { - try - { - _chain_db->open( _data_dir / "blockchain", initial_state ); - } - catch( const fc::exception& e ) - { - ilog( "Caught exception ${e} in open()", ("e", e.to_detail_string()) ); - - replay = true; - replay_reason = "exception in open()"; - } - } - - if( replay ) - { - ilog( "Replaying blockchain due to: ${reason}", ("reason", replay_reason) ); - - fc::remove_all( _data_dir / "db_version" ); - _chain_db->reindex( _data_dir / "blockchain", initial_state() ); - - const auto mode = std::ios::out | std::ios::binary | std::ios::trunc; - std::ofstream db_version( (_data_dir / "db_version").generic_string().c_str(), mode ); - std::string version_string = GRAPHENE_CURRENT_DB_VERSION; - db_version.write( version_string.c_str(), version_string.size() ); - db_version.close(); + elog( "Caught exception ${e} in open(), you might want to force a replay", ("e", e.to_detail_string()) ); + throw; } if( _options->count("force-validate") ) @@ -447,9 +397,21 @@ namespace detail { _force_validate = true; } - if( _options->count("api-access") ) - _apiaccess = fc::json::from_file( _options->at("api-access").as() ) - .as(); + if( _options->count("api-access") ) { + + if(fc::exists(_options->at("api-access").as())) + { + _apiaccess = fc::json::from_file( _options->at("api-access").as() ).as( 20 ); + ilog( "Using api access file from ${path}", + ("path", _options->at("api-access").as().string()) ); + } + else + { + elog("Failed to load file from ${path}", + ("path", _options->at("api-access").as().string())); + std::exit(EXIT_FAILURE); + } + } else { // TODO: Remove this generous default access policy @@ -992,7 +954,7 @@ void application::initialize(const fc::path& data_dir, const boost::program_opti if( fc::exists(genesis_out) ) { try { - genesis_state = fc::json::from_file(genesis_out).as(); + genesis_state = fc::json::from_file(genesis_out).as( 20 ); } catch(const fc::exception& e) { std::cerr << "Unable to parse existing genesis file:\n" << e.to_string() << "\nWould you like to replace it? [y/N] "; diff --git a/libraries/app/database_api.cpp b/libraries/app/database_api.cpp index 9aebc8f7..e692f137 100644 --- a/libraries/app/database_api.cpp +++ b/libraries/app/database_api.cpp @@ -47,9 +47,6 @@ typedef std::map< std::pair { public: @@ -103,6 +100,7 @@ class database_api_impl : public std::enable_shared_from_this vector> get_assets(const vector& asset_ids)const; vector list_assets(const string& lower_bound_symbol, uint32_t limit)const; vector> lookup_asset_symbols(const vector& symbols_or_ids)const; + uint64_t get_asset_count()const; // Peerplays vector list_sports() const; @@ -217,7 +215,7 @@ class database_api_impl : public std::enable_shared_from_this auto sub = _market_subscriptions.find( market ); if( sub != _market_subscriptions.end() ) { - queue[market].emplace_back( full_object ? obj->to_variant() : fc::variant(obj->id) ); + queue[market].emplace_back( full_object ? obj->to_variant() : fc::variant(obj->id, 1) ); } } @@ -273,7 +271,7 @@ database_api_impl::database_api_impl( graphene::chain::database& db ):_db(db) _applied_block_connection = _db.applied_block.connect([this](const signed_block&){ on_applied_block(); }); _pending_trx_connection = _db.on_pending_transaction.connect([this](const signed_transaction& trx ){ - if( _pending_trx_callback ) _pending_trx_callback( fc::variant(trx) ); + if( _pending_trx_callback ) _pending_trx_callback( fc::variant(trx, GRAPHENE_MAX_NESTED_OBJECTS) ); }); } @@ -524,6 +522,11 @@ vector> database_api::get_key_references( vector> database_api_impl::get_key_references( vector keys )const { wdump( (keys) ); + + const auto& idx = _db.get_index_type(); + const auto& aidx = dynamic_cast(idx); + const auto& refs = aidx.get_secondary_index(); + vector< vector > final_result; final_result.reserve(keys.size()); @@ -543,10 +546,6 @@ vector> database_api_impl::get_key_references( vector(); - const auto& aidx = dynamic_cast&>(idx); - const auto& refs = aidx.get_secondary_index(); - auto itr = refs.account_to_key_memberships.find(key); vector result; for( auto& a : {a1,a2,a3,a4,a5} ) @@ -563,6 +562,7 @@ vector> database_api_impl::get_key_references( vectorsecond.size() ); @@ -598,7 +598,7 @@ bool database_api_impl::is_public_key_registered(string public_key) const return false; } const auto& idx = _db.get_index_type(); - const auto& aidx = dynamic_cast&>(idx); + const auto& aidx = dynamic_cast(idx); const auto& refs = aidx.get_secondary_index(); auto itr = refs.account_to_key_memberships.find(key); bool is_known = itr != refs.account_to_key_memberships.end(); @@ -639,13 +639,17 @@ std::map database_api::get_full_accounts( const vector database_api_impl::get_full_accounts( const vector& names_or_ids, bool subscribe) { + const auto& proposal_idx = _db.get_index_type(); + const auto& pidx = dynamic_cast(proposal_idx); + const auto& proposals_by_account = pidx.get_secondary_index(); + std::map results; for (const std::string& account_name_or_id : names_or_ids) { const account_object* account = nullptr; if (std::isdigit(account_name_or_id[0])) - account = _db.find(fc::variant(account_name_or_id).as()); + account = _db.find(fc::variant(account_name_or_id, 1).as(1)); else { const auto& idx = _db.get_index_type().indices().get(); @@ -663,7 +667,6 @@ std::map database_api_impl::get_full_accounts( const subscribe_to_item( account->id ); } - // fc::mutable_variant_object full_account; full_account acnt; acnt.account = *account; acnt.statistics = account->statistics(_db); @@ -672,20 +675,11 @@ std::map database_api_impl::get_full_accounts( const acnt.lifetime_referrer_name = account->lifetime_referrer(_db).name; acnt.votes = lookup_vote_ids( vector(account->options.votes.begin(),account->options.votes.end()) ); - // Add the account itself, its statistics object, cashback balance, and referral account names - /* - full_account("account", *account)("statistics", account->statistics(_db)) - ("registrar_name", account->registrar(_db).name)("referrer_name", account->referrer(_db).name) - ("lifetime_referrer_name", account->lifetime_referrer(_db).name); - */ if (account->cashback_vb) { acnt.cashback_balance = account->cashback_balance(_db); } // Add the account's proposals - const auto& proposal_idx = _db.get_index_type(); - const auto& pidx = dynamic_cast&>(proposal_idx); - const auto& proposals_by_account = pidx.get_secondary_index(); auto required_approvals_itr = proposals_by_account._account_to_proposals.find( account->id ); if( required_approvals_itr != proposals_by_account._account_to_proposals.end() ) { @@ -696,12 +690,9 @@ std::map database_api_impl::get_full_accounts( const // Add the account's balances - auto balance_range = _db.get_index_type().indices().get().equal_range(boost::make_tuple(account->id)); - //vector balances; - std::for_each(balance_range.first, balance_range.second, - [&acnt](const account_balance_object& balance) { - acnt.balances.emplace_back(balance); - }); + const auto& balances = _db.get_index_type< primary_index< account_balance_index > >().get_secondary_index< balances_by_account_index >().get_account_balances( account->id ); + for( const auto balance : balances ) + acnt.balances.emplace_back( *balance.second ); // Add the account's vesting balances auto vesting_range = _db.get_index_type().indices().get().equal_range(account->id); @@ -773,7 +764,7 @@ vector database_api::get_account_references( account_id_type ac vector database_api_impl::get_account_references( account_id_type account_id )const { const auto& idx = _db.get_index_type(); - const auto& aidx = dynamic_cast&>(idx); + const auto& aidx = dynamic_cast(idx); const auto& refs = aidx.get_secondary_index(); auto itr = refs.account_to_account_memberships.find(account_id); vector result; @@ -854,10 +845,10 @@ vector database_api_impl::get_account_balances(account_id_type acnt, cons if (assets.empty()) { // if the caller passes in an empty list of assets, return balances for all assets the account owns - const account_balance_index& balance_index = _db.get_index_type(); - auto range = balance_index.indices().get().equal_range(boost::make_tuple(acnt)); - for (const account_balance_object& balance : boost::make_iterator_range(range.first, range.second)) - result.push_back(asset(balance.get_balance())); + const auto& balance_index = _db.get_index_type< primary_index< account_balance_index > >(); + const auto& balances = balance_index.get_secondary_index< balances_by_account_index >().get_account_balances( acnt ); + for( const auto balance : balances ) + result.push_back( balance.second->get_balance() ); } else { @@ -1013,7 +1004,7 @@ vector> database_api_impl::lookup_asset_symbols(const vec [this, &assets_by_symbol](const string& symbol_or_id) -> optional { if( !symbol_or_id.empty() && std::isdigit(symbol_or_id[0]) ) { - auto ptr = _db.find(variant(symbol_or_id).as()); + auto ptr = _db.find(variant(symbol_or_id, 1).as(1)); return ptr == nullptr? optional() : *ptr; } auto itr = assets_by_symbol.find(symbol_or_id); @@ -1022,6 +1013,15 @@ vector> database_api_impl::lookup_asset_symbols(const vec return result; } +uint64_t database_api::get_asset_count()const +{ + return my->get_asset_count(); +} + +uint64_t database_api_impl::get_asset_count()const +{ + return _db.get_index_type().indices().size(); +} //////////////////// // Lottery Assets // //////////////////// @@ -1712,7 +1712,7 @@ vector database_api_impl::lookup_vote_ids( const vector& { auto itr = committee_idx.find( id ); if( itr != committee_idx.end() ) - result.emplace_back( variant( *itr ) ); + result.emplace_back( variant( *itr, 1 ) ); else result.emplace_back( variant() ); break; @@ -1721,7 +1721,7 @@ vector database_api_impl::lookup_vote_ids( const vector& { auto itr = witness_idx.find( id ); if( itr != witness_idx.end() ) - result.emplace_back( variant( *itr ) ); + result.emplace_back( variant( *itr, 1 ) ); else result.emplace_back( variant() ); break; @@ -1730,12 +1730,12 @@ vector database_api_impl::lookup_vote_ids( const vector& { auto itr = for_worker_idx.find( id ); if( itr != for_worker_idx.end() ) { - result.emplace_back( variant( *itr ) ); + result.emplace_back( variant( *itr, 1 ) ); } else { auto itr = against_worker_idx.find( id ); if( itr != against_worker_idx.end() ) { - result.emplace_back( variant( *itr ) ); + result.emplace_back( variant( *itr, 1 ) ); } else { result.emplace_back( variant() ); @@ -1744,6 +1744,8 @@ vector database_api_impl::lookup_vote_ids( const vector& break; } case vote_id_type::VOTE_TYPE_COUNT: break; // supress unused enum value warnings + default: + FC_CAPTURE_AND_THROW( fc::out_of_range_exception, (id) ); } } return result; @@ -1852,8 +1854,8 @@ bool database_api::verify_authority( const signed_transaction& trx )const bool database_api_impl::verify_authority( const signed_transaction& trx )const { trx.verify_authority( _db.get_chain_id(), - [&]( account_id_type id ){ return &id(_db).active; }, - [&]( account_id_type id ){ return &id(_db).owner; }, + [this]( account_id_type id ){ return &id(_db).active; }, + [this]( account_id_type id ){ return &id(_db).owner; }, _db.get_global_properties().parameters.max_authority_depth ); return true; } @@ -1868,7 +1870,7 @@ bool database_api_impl::verify_account_authority( const string& name_or_id, cons FC_ASSERT( name_or_id.size() > 0); const account_object* account = nullptr; if (std::isdigit(name_or_id[0])) - account = _db.find(fc::variant(name_or_id).as()); + account = _db.find(fc::variant(name_or_id, 1).as(1)); else { const auto& idx = _db.get_index_type().indices().get(); @@ -1929,7 +1931,7 @@ struct get_required_fees_helper { asset fee = current_fee_schedule.set_fee( op, core_exchange_rate ); fc::variant result; - fc::to_variant( fee, result ); + fc::to_variant( fee, result, GRAPHENE_NET_MAX_NESTED_OBJECTS ); return result; } } @@ -1949,7 +1951,7 @@ struct get_required_fees_helper // two mutually recursive functions instead of a visitor result.first = current_fee_schedule.set_fee( proposal_create_op, core_exchange_rate ); fc::variant vresult; - fc::to_variant( result, vresult ); + fc::to_variant( result, vresult, GRAPHENE_NET_MAX_NESTED_OBJECTS ); return vresult; } @@ -2211,7 +2213,7 @@ void database_api_impl::handle_object_changed(bool force_notify, bool full_objec } else { - updates.emplace_back( id ); + updates.emplace_back( fc::variant( id, 1 ) ); } } } @@ -2255,7 +2257,7 @@ void database_api_impl::on_applied_block() auto capture_this = shared_from_this(); block_id_type block_id = _db.head_block_id(); fc::async([this,capture_this,block_id](){ - _block_applied_callback(fc::variant(block_id)); + _block_applied_callback(fc::variant(block_id, 1)); }); } @@ -2296,7 +2298,7 @@ void database_api_impl::on_applied_block() { auto itr = _market_subscriptions.find(item.first); if(itr != _market_subscriptions.end()) - itr->second(fc::variant(item.second)); + itr->second(fc::variant(item.second, GRAPHENE_NET_MAX_NESTED_OBJECTS)); } }); } diff --git a/libraries/app/include/graphene/app/database_api.hpp b/libraries/app/include/graphene/app/database_api.hpp index 6f90938d..78a9ca1f 100644 --- a/libraries/app/include/graphene/app/database_api.hpp +++ b/libraries/app/include/graphene/app/database_api.hpp @@ -342,6 +342,12 @@ class database_api * This function has semantics identical to @ref get_objects */ vector> lookup_asset_symbols(const vector& symbols_or_ids)const; + + /** + * @brief Get assets count + * @return The assets count + */ + uint64_t get_asset_count()const; //////////////////// // Lottery Assets // @@ -727,6 +733,7 @@ FC_API(graphene::app::database_api, (get_assets) (list_assets) (lookup_asset_symbols) + (get_asset_count) // Peerplays (list_sports) diff --git a/libraries/app/include/graphene/app/plugin.hpp b/libraries/app/include/graphene/app/plugin.hpp index 5d67f6e3..45336f67 100644 --- a/libraries/app/include/graphene/app/plugin.hpp +++ b/libraries/app/include/graphene/app/plugin.hpp @@ -123,16 +123,24 @@ class plugin : public abstract_plugin /// @group Some useful tools for boost::program_options arguments using vectors of JSON strings /// @{ template -T dejsonify(const string& s) +T dejsonify(const string& s, uint32_t max_depth) { - return fc::json::from_string(s).as(); + return fc::json::from_string(s).as(max_depth); +} + +namespace impl { + template + T dejsonify( const string& s ) + { + return graphene::app::dejsonify( s, GRAPHENE_MAX_NESTED_OBJECTS ); + } } #define DEFAULT_VALUE_VECTOR(value) default_value({fc::json::to_string(value)}, fc::json::to_string(value)) #define LOAD_VALUE_SET(options, name, container, type) \ if( options.count(name) ) { \ const std::vector& ops = options[name].as>(); \ - std::transform(ops.begin(), ops.end(), std::inserter(container, container.end()), &graphene::app::dejsonify); \ + std::transform(ops.begin(), ops.end(), std::inserter(container, container.end()), &graphene::app::impl::dejsonify); \ } /// @} diff --git a/libraries/chain/account_object.cpp b/libraries/chain/account_object.cpp index 90d97692..e51e1705 100644 --- a/libraries/chain/account_object.cpp +++ b/libraries/chain/account_object.cpp @@ -119,9 +119,9 @@ set account_member_index::get_account_members(const account_obj result.insert(auth.first); return result; } -set account_member_index::get_key_members(const account_object& a)const +set account_member_index::get_key_members(const account_object& a)const { - set result; + set result; for( auto auth : a.owner.key_auths ) result.insert(auth.first); for( auto auth : a.active.key_auths ) @@ -213,7 +213,7 @@ void account_member_index::object_modified(const object& after) { - set after_key_members = get_key_members(a); + set after_key_members = get_key_members(a); vector removed; removed.reserve(before_key_members.size()); std::set_difference(before_key_members.begin(), before_key_members.end(), @@ -267,4 +267,54 @@ void account_referrer_index::object_modified( const object& after ) { } +const uint8_t balances_by_account_index::bits = 20; +const uint64_t balances_by_account_index::mask = (1ULL << balances_by_account_index::bits) - 1; + +void balances_by_account_index::object_inserted( const object& obj ) +{ + const auto& abo = dynamic_cast< const account_balance_object& >( obj ); + while( balances.size() < (abo.owner.instance.value >> bits) + 1 ) + { + balances.reserve( (abo.owner.instance.value >> bits) + 1 ); + balances.resize( balances.size() + 1 ); + balances.back().resize( 1ULL << bits ); + } + balances[abo.owner.instance.value >> bits][abo.owner.instance.value & mask][abo.asset_type] = &abo; +} + +void balances_by_account_index::object_removed( const object& obj ) +{ + const auto& abo = dynamic_cast< const account_balance_object& >( obj ); + if( balances.size() < (abo.owner.instance.value >> bits) + 1 ) return; + balances[abo.owner.instance.value >> bits][abo.owner.instance.value & mask].erase( abo.asset_type ); +} + +void balances_by_account_index::about_to_modify( const object& before ) +{ + ids_being_modified.emplace( before.id ); +} + +void balances_by_account_index::object_modified( const object& after ) +{ + FC_ASSERT( ids_being_modified.top() == after.id, "Modification of ID is not supported!"); + ids_being_modified.pop(); +} + +const map< asset_id_type, const account_balance_object* >& balances_by_account_index::get_account_balances( const account_id_type& acct )const +{ + static const map< asset_id_type, const account_balance_object* > _empty; + + if( balances.size() < (acct.instance.value >> bits) + 1 ) return _empty; + return balances[acct.instance.value >> bits][acct.instance.value & mask]; +} + +const account_balance_object* balances_by_account_index::get_account_balance( const account_id_type& acct, const asset_id_type& asset )const +{ + if( balances.size() < (acct.instance.value >> bits) + 1 ) return nullptr; + const auto& mine = balances[acct.instance.value >> bits][acct.instance.value & mask]; + const auto itr = mine.find( asset ); + if( mine.end() == itr ) return nullptr; + return itr->second; +} + } } // graphene::chain diff --git a/libraries/chain/betting_market_group_object.cpp b/libraries/chain/betting_market_group_object.cpp index 71135e07..2ac7d7a4 100644 --- a/libraries/chain/betting_market_group_object.cpp +++ b/libraries/chain/betting_market_group_object.cpp @@ -543,35 +543,35 @@ void betting_market_group_object::dispatch_new_status(database& db, betting_mark namespace fc { // Manually reflect betting_market_group_object to variant to properly reflect "state" - void to_variant(const graphene::chain::betting_market_group_object& betting_market_group_obj, fc::variant& v) + void to_variant(const graphene::chain::betting_market_group_object& betting_market_group_obj, fc::variant& v, uint32_t max_depth) { fc::mutable_variant_object o; - o("id", betting_market_group_obj.id) - ("description", betting_market_group_obj.description) - ("event_id", betting_market_group_obj.event_id) - ("rules_id", betting_market_group_obj.rules_id) - ("asset_id", betting_market_group_obj.asset_id) - ("total_matched_bets_amount", betting_market_group_obj.total_matched_bets_amount) - ("never_in_play", betting_market_group_obj.never_in_play) - ("delay_before_settling", betting_market_group_obj.delay_before_settling) - ("settling_time", betting_market_group_obj.settling_time) - ("status", betting_market_group_obj.get_status()); + o("id", fc::variant(betting_market_group_obj.id, max_depth)) + ("description", fc::variant(betting_market_group_obj.description, max_depth)) + ("event_id", fc::variant(betting_market_group_obj.event_id, max_depth)) + ("rules_id", fc::variant(betting_market_group_obj.rules_id, max_depth)) + ("asset_id", fc::variant(betting_market_group_obj.asset_id, max_depth)) + ("total_matched_bets_amount", fc::variant(betting_market_group_obj.total_matched_bets_amount, max_depth)) + ("never_in_play", fc::variant(betting_market_group_obj.never_in_play, max_depth)) + ("delay_before_settling", fc::variant(betting_market_group_obj.delay_before_settling, max_depth)) + ("settling_time", fc::variant(betting_market_group_obj.settling_time, max_depth)) + ("status", fc::variant(betting_market_group_obj.get_status(), max_depth)); v = o; } // Manually reflect betting_market_group_object to variant to properly reflect "state" - void from_variant(const fc::variant& v, graphene::chain::betting_market_group_object& betting_market_group_obj) + void from_variant(const fc::variant& v, graphene::chain::betting_market_group_object& betting_market_group_obj, uint32_t max_depth) { - betting_market_group_obj.id = v["id"].as(); - betting_market_group_obj.description = v["description"].as(); - betting_market_group_obj.event_id = v["event_id"].as(); - betting_market_group_obj.asset_id = v["asset_id"].as(); - betting_market_group_obj.total_matched_bets_amount = v["total_matched_bets_amount"].as(); - betting_market_group_obj.never_in_play = v["never_in_play"].as(); - betting_market_group_obj.delay_before_settling = v["delay_before_settling"].as(); - betting_market_group_obj.settling_time = v["settling_time"].as>(); - graphene::chain::betting_market_group_status status = v["status"].as(); + betting_market_group_obj.id = v["id"].as( max_depth ); + betting_market_group_obj.description = v["description"].as( max_depth ); + betting_market_group_obj.event_id = v["event_id"].as( max_depth ); + betting_market_group_obj.asset_id = v["asset_id"].as( max_depth ); + betting_market_group_obj.total_matched_bets_amount = v["total_matched_bets_amount"].as( max_depth ); + betting_market_group_obj.never_in_play = v["never_in_play"].as( max_depth ); + betting_market_group_obj.delay_before_settling = v["delay_before_settling"].as( max_depth ); + betting_market_group_obj.settling_time = v["settling_time"].as>( max_depth ); + graphene::chain::betting_market_group_status status = v["status"].as( max_depth ); const_cast(betting_market_group_obj.my->state_machine.current_state())[0] = (int)status; } } //end namespace fc diff --git a/libraries/chain/betting_market_object.cpp b/libraries/chain/betting_market_object.cpp index cb0e006e..d5efd56c 100644 --- a/libraries/chain/betting_market_object.cpp +++ b/libraries/chain/betting_market_object.cpp @@ -468,28 +468,28 @@ void betting_market_object::on_canceled_event(database& db) namespace fc { // Manually reflect betting_market_object to variant to properly reflect "state" - void to_variant(const graphene::chain::betting_market_object& event_obj, fc::variant& v) + void to_variant(const graphene::chain::betting_market_object& event_obj, fc::variant& v, uint32_t max_depth) { fc::mutable_variant_object o; - o("id", event_obj.id) - ("group_id", event_obj.group_id) - ("description", event_obj.description) - ("payout_condition", event_obj.payout_condition) - ("resolution", event_obj.resolution) - ("status", event_obj.get_status()); + o("id", fc::variant(event_obj.id, max_depth) ) + ("group_id", fc::variant(event_obj.group_id, max_depth)) + ("description", fc::variant(event_obj.description, max_depth)) + ("payout_condition", fc::variant(event_obj.payout_condition, max_depth)) + ("resolution", fc::variant(event_obj.resolution, max_depth)) + ("status", fc::variant(event_obj.get_status(), max_depth)); v = o; } // Manually reflect betting_market_object to variant to properly reflect "state" - void from_variant(const fc::variant& v, graphene::chain::betting_market_object& event_obj) + void from_variant(const fc::variant& v, graphene::chain::betting_market_object& event_obj, uint32_t max_depth) { - event_obj.id = v["id"].as(); - event_obj.group_id = v["name"].as(); - event_obj.description = v["description"].as(); - event_obj.payout_condition = v["payout_condition"].as(); - event_obj.resolution = v["resolution"].as>(); - graphene::chain::betting_market_status status = v["status"].as(); + event_obj.id = v["id"].as( max_depth ); + event_obj.group_id = v["name"].as( max_depth ); + event_obj.description = v["description"].as( max_depth ); + event_obj.payout_condition = v["payout_condition"].as( max_depth ); + event_obj.resolution = v["resolution"].as>( max_depth ); + graphene::chain::betting_market_status status = v["status"].as( max_depth ); const_cast(event_obj.my->state_machine.current_state())[0] = (int)status; } } //end namespace fc diff --git a/libraries/chain/block_database.cpp b/libraries/chain/block_database.cpp index 214459f0..3dcdcba4 100644 --- a/libraries/chain/block_database.cpp +++ b/libraries/chain/block_database.cpp @@ -45,14 +45,15 @@ void block_database::open( const fc::path& dbdir ) _block_num_to_pos.exceptions(std::ios_base::failbit | std::ios_base::badbit); _blocks.exceptions(std::ios_base::failbit | std::ios_base::badbit); - if( !fc::exists( dbdir/"index" ) ) + _index_filename = dbdir / "index"; + if( !fc::exists( _index_filename ) ) { - _block_num_to_pos.open( (dbdir/"index").generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out | std::fstream::trunc); + _block_num_to_pos.open( _index_filename.generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out | std::fstream::trunc); _blocks.open( (dbdir/"blocks").generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out | std::fstream::trunc); } else { - _block_num_to_pos.open( (dbdir/"index").generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out ); + _block_num_to_pos.open( _index_filename.generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out ); _blocks.open( (dbdir/"blocks").generic_string().c_str(), std::fstream::binary | std::fstream::in | std::fstream::out ); } } FC_CAPTURE_AND_RETHROW( (dbdir) ) } @@ -121,7 +122,7 @@ bool block_database::contains( const block_id_type& id )const index_entry e; auto index_pos = sizeof(e)*block_header::num_from_id(id); _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); - if ( _block_num_to_pos.tellg() <= index_pos ) + if ( _block_num_to_pos.tellg() < index_pos + sizeof(e) ) return false; _block_num_to_pos.seekg( index_pos ); _block_num_to_pos.read( (char*)&e, sizeof(e) ); @@ -206,34 +207,47 @@ optional block_database::fetch_by_number( uint32_t block_num )cons return optional(); } -optional block_database::last()const -{ +optional block_database::last_index_entry()const { try { index_entry e; + _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); + std::streampos pos = _block_num_to_pos.tellg(); + if( pos < sizeof(index_entry) ) + return optional(); - if( _block_num_to_pos.tellp() < sizeof(index_entry) ) - return optional(); + pos -= pos % sizeof(index_entry); - _block_num_to_pos.seekg( -sizeof(index_entry), _block_num_to_pos.end ); - _block_num_to_pos.read( (char*)&e, sizeof(e) ); - uint64_t pos = _block_num_to_pos.tellg(); - while( e.block_size == 0 && pos > 0 ) + _blocks.seekg( 0, _block_num_to_pos.end ); + const std::streampos blocks_size = _blocks.tellg(); + while( pos > 0 ) { pos -= sizeof(index_entry); _block_num_to_pos.seekg( pos ); _block_num_to_pos.read( (char*)&e, sizeof(e) ); + if( _block_num_to_pos.gcount() == sizeof(e) && e.block_size > 0 + && e.block_pos + e.block_size <= blocks_size ) + try + { + vector data( e.block_size ); + _blocks.seekg( e.block_pos ); + _blocks.read( data.data(), e.block_size ); + if( _blocks.gcount() == e.block_size ) + { + const signed_block block = fc::raw::unpack(data); + if( block.id() == e.block_id ) + return e; + } + } + catch (const fc::exception&) + { + } + catch (const std::exception&) + { + } + fc::resize_file( _index_filename, pos ); } - - if( e.block_size == 0 ) - return optional(); - - vector data( e.block_size ); - _blocks.seekg( e.block_pos ); - _blocks.read( data.data(), e.block_size ); - auto result = fc::raw::unpack(data); - return result; } catch (const fc::exception&) { @@ -241,42 +255,21 @@ optional block_database::last()const catch (const std::exception&) { } + return optional(); +} + +optional block_database::last()const +{ + optional entry = last_index_entry(); + if( entry.valid() ) return fetch_by_number( block_header::num_from_id(entry->block_id) ); return optional(); } optional block_database::last_id()const { - try - { - index_entry e; - _block_num_to_pos.seekg( 0, _block_num_to_pos.end ); - - if( _block_num_to_pos.tellp() < sizeof(index_entry) ) - return optional(); - - _block_num_to_pos.seekg( -sizeof(index_entry), _block_num_to_pos.end ); - _block_num_to_pos.read( (char*)&e, sizeof(e) ); - uint64_t pos = _block_num_to_pos.tellg(); - while( e.block_size == 0 && pos > 0 ) - { - pos -= sizeof(index_entry); - _block_num_to_pos.seekg( pos ); - _block_num_to_pos.read( (char*)&e, sizeof(e) ); - } - - if( e.block_size == 0 ) - return optional(); - - return e.block_id; - } - catch (const fc::exception&) - { - } - catch (const std::exception&) - { - } + optional entry = last_index_entry(); + if( entry.valid() ) return entry->block_id; return optional(); } - } } diff --git a/libraries/chain/db_balance.cpp b/libraries/chain/db_balance.cpp index 0b5e2c02..7a46df17 100644 --- a/libraries/chain/db_balance.cpp +++ b/libraries/chain/db_balance.cpp @@ -34,11 +34,11 @@ namespace graphene { namespace chain { asset database::get_balance(account_id_type owner, asset_id_type asset_id) const { - auto& index = get_index_type().indices().get(); - auto itr = index.find(boost::make_tuple(owner, asset_id)); - if( itr == index.end() ) + auto& index = get_index_type< primary_index< account_balance_index > >().get_secondary_index(); + auto abo = index.get_account_balance( owner, asset_id ); + if( !abo ) return asset(0, asset_id); - return itr->get_balance(); + return abo->get_balance(); } asset database::get_balance(const account_object& owner, const asset_object& asset_obj) const @@ -65,9 +65,9 @@ void database::adjust_balance(account_id_type account, asset delta ) if( delta.amount == 0 ) return; - auto& index = get_index_type().indices().get(); - auto itr = index.find(boost::make_tuple(account, delta.asset_id)); - if(itr == index.end()) + auto& index = get_index_type< primary_index< account_balance_index > >().get_secondary_index(); + auto abo = index.get_account_balance( account, delta.asset_id ); + if( !abo ) { FC_ASSERT( delta.amount > 0, "Insufficient Balance: ${a}'s balance of ${b} is less than required ${r}", ("a",account(*this).name) @@ -80,8 +80,9 @@ void database::adjust_balance(account_id_type account, asset delta ) }); } else { if( delta.amount < 0 ) - FC_ASSERT( itr->get_balance() >= -delta, "Insufficient Balance: ${a}'s balance of ${b} is less than required ${r}", ("a",account(*this).name)("b",to_pretty_string(itr->get_balance()))("r",to_pretty_string(-delta))); - modify(*itr, [delta](account_balance_object& b) { + FC_ASSERT( abo->get_balance() >= -delta, "Insufficient Balance: ${a}'s balance of ${b} is less than required ${r}", + ("a",account(*this).name)("b",to_pretty_string(abo->get_balance()))("r",to_pretty_string(-delta))); + modify(*abo, [delta](account_balance_object& b) { b.adjust_balance(delta); }); } diff --git a/libraries/chain/db_block.cpp b/libraries/chain/db_block.cpp index e9124594..1ad84fa0 100644 --- a/libraries/chain/db_block.cpp +++ b/libraries/chain/db_block.cpp @@ -215,12 +215,15 @@ bool database::_push_block(const signed_block& new_block) // pop blocks until we hit the forked block while( head_block_id() != branches.second.back()->data.previous ) + { + ilog( "popping block #${n} ${id}", ("n",head_block_num())("id",head_block_id()) ); pop_block(); + } // push all blocks on the new fork for( auto ritr = branches.first.rbegin(); ritr != branches.first.rend(); ++ritr ) { - ilog( "pushing blocks from fork ${n} ${id}", ("n",(*ritr)->data.block_num())("id",(*ritr)->data.id()) ); + ilog( "pushing block from fork #${n} ${id}", ("n",(*ritr)->data.block_num())("id",(*ritr)->id) ); optional except; try { undo_database::session session = _undo_db.start_undo_session(); @@ -235,21 +238,27 @@ bool database::_push_block(const signed_block& new_block) // remove the rest of branches.first from the fork_db, those blocks are invalid while( ritr != branches.first.rend() ) { - _fork_db.remove( (*ritr)->data.id() ); + ilog( "removing block from fork_db #${n} ${id}", ("n",(*ritr)->data.block_num())("id",(*ritr)->id) ); + _fork_db.remove( (*ritr)->id ); ++ritr; } _fork_db.set_head( branches.second.front() ); // pop all blocks from the bad fork while( head_block_id() != branches.second.back()->data.previous ) - pop_block(); - - // restore all blocks from the good fork - for( auto ritr = branches.second.rbegin(); ritr != branches.second.rend(); ++ritr ) { + ilog( "popping block #${n} ${id}", ("n",head_block_num())("id",head_block_id()) ); + pop_block(); + } + + ilog( "Switching back to fork: ${id}", ("id",branches.second.front()->data.id()) ); + // restore all blocks from the good fork + for( auto ritr2 = branches.second.rbegin(); ritr2 != branches.second.rend(); ++ritr2 ) + { + ilog( "pushing block #${n} ${id}", ("n",(*ritr2)->data.block_num())("id",(*ritr2)->id) ); auto session = _undo_db.start_undo_session(); - apply_block( (*ritr)->data, skip ); - _block_id_to_block.store( new_block.id(), (*ritr)->data ); + apply_block( (*ritr2)->data, skip ); + _block_id_to_block.store( (*ritr2)->id, (*ritr2)->data ); session.commit(); } throw *except; @@ -327,6 +336,8 @@ processed_transaction database::validate_transaction( const signed_transaction& processed_transaction database::push_proposal(const proposal_object& proposal) { try { + FC_ASSERT( _undo_db.size() < _undo_db.max_size(), "Undo database is full!" ); + transaction_evaluation_state eval_state(this); eval_state._is_proposed_trx = true; @@ -489,7 +500,7 @@ signed_block database::_generate_block( FC_ASSERT( fc::raw::pack_size(pending_block) <= get_global_properties().parameters.maximum_block_size ); } - push_block( pending_block, skip ); + push_block( pending_block, skip | skip_transaction_signatures ); // skip authority check when pushing self-generated blocks return pending_block; } FC_CAPTURE_AND_RETHROW( (witness_id) ) } @@ -506,7 +517,6 @@ void database::pop_block() GRAPHENE_ASSERT( head_block.valid(), pop_empty_chain, "there are no blocks to pop" ); _fork_db.pop_block(); - _block_id_to_block.remove( head_id ); pop_undo(); _popped_tx.insert( _popped_tx.begin(), head_block->transactions.begin(), head_block->transactions.end() ); @@ -663,6 +673,19 @@ processed_transaction database::apply_transaction(const signed_transaction& trx, return result; } +class undo_size_restorer { + public: + undo_size_restorer( undo_database& db ) : _db( db ), old_max( db.max_size() ) { + _db.set_max_size( old_max * 2 ); + } + ~undo_size_restorer() { + _db.set_max_size( old_max ); + } + private: + undo_database& _db; + size_t old_max; +}; + processed_transaction database::_apply_transaction(const signed_transaction& trx) { try { uint32_t skip = get_node_properties().skip_flags; @@ -672,9 +695,14 @@ processed_transaction database::_apply_transaction(const signed_transaction& trx auto& trx_idx = get_mutable_index_type(); const chain_id_type& chain_id = get_chain_id(); - auto trx_id = trx.id(); - FC_ASSERT( (skip & skip_transaction_dupe_check) || - trx_idx.indices().get().find(trx_id) == trx_idx.indices().get().end() ); + transaction_id_type trx_id; + + if( !(skip & skip_transaction_dupe_check) ) + { + trx_id = trx.id(); + FC_ASSERT( trx_idx.indices().get().find(trx_id) == trx_idx.indices().get().end() ); + } + transaction_evaluation_state eval_state(this); const chain_parameters& chain_parameters = get_global_properties().parameters; eval_state._trx = &trx; @@ -708,7 +736,7 @@ processed_transaction database::_apply_transaction(const signed_transaction& trx //Insert transaction into unique transactions database. if( !(skip & skip_transaction_dupe_check) ) { - create([&](transaction_object& transaction) { + create([&trx_id,&trx](transaction_object& transaction) { transaction.trx_id = trx_id; transaction.trx = trx; }); @@ -716,6 +744,7 @@ processed_transaction database::_apply_transaction(const signed_transaction& trx eval_state.operation_results.reserve(trx.operations.size()); + const undo_size_restorer undo_guard( _undo_db ); //Finally process the operations processed_transaction ptrx(trx); _current_op_in_trx = 0; @@ -729,9 +758,9 @@ processed_transaction database::_apply_transaction(const signed_transaction& trx ptrx.operation_results = std::move(eval_state.operation_results); //Make sure the temp account has no non-zero balances - const auto& index = get_index_type().indices().get(); - auto range = index.equal_range( boost::make_tuple( GRAPHENE_TEMP_ACCOUNT ) ); - std::for_each(range.first, range.second, [](const account_balance_object& b) { FC_ASSERT(b.balance == 0); }); + const auto& balances = get_index_type< primary_index< account_balance_index > >().get_secondary_index< balances_by_account_index >().get_account_balances( GRAPHENE_TEMP_ACCOUNT ); + for( const auto b : balances ) + FC_ASSERT(b.second->balance == 0); return ptrx; } FC_CAPTURE_AND_RETHROW( (trx) ) } diff --git a/libraries/chain/db_debug.cpp b/libraries/chain/db_debug.cpp index aa91fd44..0fa5eb58 100644 --- a/libraries/chain/db_debug.cpp +++ b/libraries/chain/db_debug.cpp @@ -118,10 +118,10 @@ void debug_apply_update( database& db, const fc::variant_object& vo ) auto it_id = vo.find("id"); FC_ASSERT( it_id != vo.end() ); - from_variant( it_id->value(), oid ); + from_variant( it_id->value(), oid, GRAPHENE_MAX_NESTED_OBJECTS ); action = ( vo.size() == 1 ) ? db_action_delete : db_action_write; - from_variant( vo["id"], oid ); + from_variant( vo["id"], oid, GRAPHENE_MAX_NESTED_OBJECTS ); if( vo.size() == 1 ) action = db_action_delete; auto it_action = vo.find("_action" ); @@ -143,25 +143,19 @@ void debug_apply_update( database& db, const fc::variant_object& vo ) switch( action ) { case db_action_create: - /* - idx.create( [&]( object& obj ) - { - idx.object_from_variant( vo, obj ); - } ); - */ FC_ASSERT( false ); break; case db_action_write: db.modify( db.get_object( oid ), [&]( object& obj ) { idx.object_default( obj ); - idx.object_from_variant( vo, obj ); + idx.object_from_variant( vo, obj, GRAPHENE_MAX_NESTED_OBJECTS ); } ); break; case db_action_update: db.modify( db.get_object( oid ), [&]( object& obj ) { - idx.object_from_variant( vo, obj ); + idx.object_from_variant( vo, obj, GRAPHENE_MAX_NESTED_OBJECTS ); } ); break; case db_action_delete: diff --git a/libraries/chain/db_init.cpp b/libraries/chain/db_init.cpp index 99343682..b1fa7424 100644 --- a/libraries/chain/db_init.cpp +++ b/libraries/chain/db_init.cpp @@ -251,15 +251,15 @@ void database::initialize_indexes() _undo_db.set_max_size( GRAPHENE_MIN_UNDO_HISTORY ); //Protocol object indexes - add_index< primary_index >(); + add_index< primary_index >(); // 8192 assets per chunk add_index< primary_index >(); - auto acnt_index = add_index< primary_index >(); + auto acnt_index = add_index< primary_index >(); // ~1 million accounts per chunk acnt_index->add_secondary_index(); acnt_index->add_secondary_index(); - add_index< primary_index >(); - add_index< primary_index >(); + add_index< primary_index >(); // 256 members per chunk + add_index< primary_index >(); // 1024 witnesses per chunk add_index< primary_index >(); add_index< primary_index >(); @@ -287,8 +287,11 @@ void database::initialize_indexes() //Implementation object indexes add_index< primary_index >(); - add_index< primary_index >(); - add_index< primary_index >(); + + auto bal_idx = add_index< primary_index >(); + bal_idx->add_secondary_index(); + + add_index< primary_index >(); // 8192 add_index< primary_index >(); add_index< primary_index> >(); add_index< primary_index> >(); @@ -739,7 +742,7 @@ void database::init_genesis(const genesis_state_type& genesis_state) vbo.owner = get_account_id(account.name); vbo.balance = asset(vesting_balance.amount, get_asset_id(vesting_balance.asset_symbol)); if (vesting_balance.policy_type == "linear") { - auto initial_linear_vesting_policy = vesting_balance.policy.as(); + auto initial_linear_vesting_policy = vesting_balance.policy.as( 20 ); linear_vesting_policy new_vesting_policy; new_vesting_policy.begin_timestamp = initial_linear_vesting_policy.begin_timestamp; new_vesting_policy.vesting_cliff_seconds = initial_linear_vesting_policy.vesting_cliff_seconds; @@ -747,7 +750,7 @@ void database::init_genesis(const genesis_state_type& genesis_state) new_vesting_policy.begin_balance = initial_linear_vesting_policy.begin_balance; vbo.policy = new_vesting_policy; } else if (vesting_balance.policy_type == "cdd") { - auto initial_cdd_vesting_policy = vesting_balance.policy.as(); + auto initial_cdd_vesting_policy = vesting_balance.policy.as( 20 ); cdd_vesting_policy new_vesting_policy; new_vesting_policy.vesting_seconds = initial_cdd_vesting_policy.vesting_seconds; new_vesting_policy.coin_seconds_earned = initial_cdd_vesting_policy.coin_seconds_earned; diff --git a/libraries/chain/db_maint.cpp b/libraries/chain/db_maint.cpp index aa492097..3ec84d14 100644 --- a/libraries/chain/db_maint.cpp +++ b/libraries/chain/db_maint.cpp @@ -621,7 +621,7 @@ void distribute_fba_balances( database& db ) void create_buyback_orders( database& db ) { const auto& bbo_idx = db.get_index_type< buyback_index >().indices().get(); - const auto& bal_idx = db.get_index_type< account_balance_index >().indices().get< by_account_asset >(); + const auto& bal_idx = db.get_index_type< primary_index< account_balance_index > >().get_secondary_index< balances_by_account_index >(); for( const buyback_object& bbo : bbo_idx ) { @@ -629,7 +629,6 @@ void create_buyback_orders( database& db ) assert( asset_to_buy.buyback_account.valid() ); const account_object& buyback_account = (*(asset_to_buy.buyback_account))(db); - asset_id_type next_asset = asset_id_type(); if( !buyback_account.allowed_assets.valid() ) { @@ -637,16 +636,11 @@ void create_buyback_orders( database& db ) continue; } - while( true ) + for( const auto& entry : bal_idx.get_account_balances( buyback_account.id ) ) { - auto it = bal_idx.lower_bound( boost::make_tuple( buyback_account.id, next_asset ) ); - if( it == bal_idx.end() ) - break; - if( it->owner != buyback_account.id ) - break; + const auto* it = entry.second; asset_id_type asset_to_sell = it->asset_type; share_type amount_to_sell = it->balance; - next_asset = asset_to_sell + 1; if( asset_to_sell == asset_to_buy.id ) continue; if( amount_to_sell == 0 ) @@ -740,8 +734,10 @@ void schedule_pending_dividend_balances(database& db, { try { dlog("Processing dividend payments for dividend holder asset type ${holder_asset} at time ${t}", ("holder_asset", dividend_holder_asset_obj.symbol)("t", db.head_block_time())); + auto balance_by_acc_index = db.get_index_type< primary_index< account_balance_index > >().get_secondary_index< balances_by_account_index >(); auto current_distribution_account_balance_range = - balance_index.indices().get().equal_range(boost::make_tuple(dividend_data.dividend_distribution_account)); + //balance_index.indices().get().equal_range(boost::make_tuple(dividend_data.dividend_distribution_account)); + balance_by_acc_index.get_account_balances(dividend_data.dividend_distribution_account); auto previous_distribution_account_balance_range = distributed_dividend_balance_index.indices().get().equal_range(boost::make_tuple(dividend_holder_asset_obj.id)); // the current range is now all current balances for the distribution account, sorted by asset_type @@ -789,10 +785,10 @@ void schedule_pending_dividend_balances(database& db, } #endif - auto current_distribution_account_balance_iter = current_distribution_account_balance_range.first; + auto current_distribution_account_balance_iter = current_distribution_account_balance_range.begin(); auto previous_distribution_account_balance_iter = previous_distribution_account_balance_range.first; dlog("Current balances in distribution account: ${current}, Previous balances: ${previous}", - ("current", (int64_t)std::distance(current_distribution_account_balance_range.first, current_distribution_account_balance_range.second)) + ("current", (int64_t)std::distance(current_distribution_account_balance_range.begin(), current_distribution_account_balance_range.end())) ("previous", (int64_t)std::distance(previous_distribution_account_balance_range.first, previous_distribution_account_balance_range.second))); // when we pay out the dividends to the holders, we need to know the total balance of the dividend asset in all @@ -808,7 +804,7 @@ void schedule_pending_dividend_balances(database& db, total_balance_of_dividend_asset += itr->second; } // loop through all of the assets currently or previously held in the distribution account - while (current_distribution_account_balance_iter != current_distribution_account_balance_range.second || + while (current_distribution_account_balance_iter != current_distribution_account_balance_range.end() || previous_distribution_account_balance_iter != previous_distribution_account_balance_range.second) { try @@ -819,15 +815,15 @@ void schedule_pending_dividend_balances(database& db, asset_id_type payout_asset_type; if (previous_distribution_account_balance_iter == previous_distribution_account_balance_range.second || - current_distribution_account_balance_iter->asset_type < previous_distribution_account_balance_iter->dividend_payout_asset_type) + current_distribution_account_balance_iter->second->asset_type < previous_distribution_account_balance_iter->dividend_payout_asset_type) { // there are no more previous balances or there is no previous balance for this particular asset type - payout_asset_type = current_distribution_account_balance_iter->asset_type; - current_balance = current_distribution_account_balance_iter->balance; + payout_asset_type = current_distribution_account_balance_iter->second->asset_type; + current_balance = current_distribution_account_balance_iter->second->balance; idump((payout_asset_type)(current_balance)); } - else if (current_distribution_account_balance_iter == current_distribution_account_balance_range.second || - previous_distribution_account_balance_iter->dividend_payout_asset_type < current_distribution_account_balance_iter->asset_type) + else if (current_distribution_account_balance_iter == current_distribution_account_balance_range.end() || + previous_distribution_account_balance_iter->dividend_payout_asset_type < current_distribution_account_balance_iter->second->asset_type) { // there are no more current balances or there is no current balance for this particular previous asset type payout_asset_type = previous_distribution_account_balance_iter->dividend_payout_asset_type; @@ -837,8 +833,8 @@ void schedule_pending_dividend_balances(database& db, else { // we have both a previous and a current balance for this asset type - payout_asset_type = current_distribution_account_balance_iter->asset_type; - current_balance = current_distribution_account_balance_iter->balance; + payout_asset_type = current_distribution_account_balance_iter->second->asset_type; + current_balance = current_distribution_account_balance_iter->second->balance; previous_balance = previous_distribution_account_balance_iter->balance_at_last_maintenance_interval; idump((payout_asset_type)(current_balance)(previous_balance)); } @@ -1043,10 +1039,10 @@ void schedule_pending_dividend_balances(database& db, // iterate if (previous_distribution_account_balance_iter == previous_distribution_account_balance_range.second || - current_distribution_account_balance_iter->asset_type < previous_distribution_account_balance_iter->dividend_payout_asset_type) + current_distribution_account_balance_iter->second->asset_type < previous_distribution_account_balance_iter->dividend_payout_asset_type) ++current_distribution_account_balance_iter; - else if (current_distribution_account_balance_iter == current_distribution_account_balance_range.second || - previous_distribution_account_balance_iter->dividend_payout_asset_type < current_distribution_account_balance_iter->asset_type) + else if (current_distribution_account_balance_iter == current_distribution_account_balance_range.end() || + previous_distribution_account_balance_iter->dividend_payout_asset_type < current_distribution_account_balance_iter->second->asset_type) ++previous_distribution_account_balance_iter; else { @@ -1066,6 +1062,7 @@ void process_dividend_assets(database& db) ilog("In process_dividend_assets time ${time}", ("time", db.head_block_time())); const account_balance_index& balance_index = db.get_index_type(); + //const auto& balance_index = db.get_index_type< primary_index< account_balance_index > >().get_secondary_index< balances_by_account_index >(); const vesting_balance_index& vbalance_index = db.get_index_type(); const total_distributed_dividend_balance_object_index& distributed_dividend_balance_index = db.get_index_type(); const pending_dividend_payout_balance_for_holder_object_index& pending_payout_balance_index = db.get_index_type(); @@ -1090,10 +1087,10 @@ void process_dividend_assets(database& db) ("holder_asset", dividend_holder_asset_obj.symbol)); #ifndef NDEBUG // dump balances before the payouts for debugging - const auto& balance_idx = db.get_index_type().indices().get(); - auto holder_account_balance_range = balance_idx.equal_range(boost::make_tuple(dividend_data.dividend_distribution_account)); - for (const account_balance_object& holder_balance_object : boost::make_iterator_range(holder_account_balance_range.first, holder_account_balance_range.second)) - ilog(" Current balance: ${asset}", ("asset", asset(holder_balance_object.balance, holder_balance_object.asset_type))); + const auto& balance_index = db.get_index_type< primary_index< account_balance_index > >(); + const auto& balances = balance_index.get_secondary_index< balances_by_account_index >().get_account_balances( dividend_data.dividend_distribution_account ); + for( const auto balance : balances ) + ilog(" Current balance: ${asset}", ("asset", asset(balance.second->balance, balance.second->asset_type))); #endif // when we do the payouts, we first increase the balances in all of the receiving accounts diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp index a0a78682..029a55d4 100644 --- a/libraries/chain/db_management.cpp +++ b/libraries/chain/db_management.cpp @@ -47,33 +47,86 @@ database::~database() clear_pending(); } -void database::reindex(fc::path data_dir, const genesis_state_type& initial_allocation) -{ try { - ilog( "reindexing blockchain" ); - wipe(data_dir, false); - open(data_dir, [&initial_allocation]{return initial_allocation;}); +// Right now, we leave undo_db enabled when replaying when the bookie plugin is +// enabled. It depends on new/changed/removed object notifications, and those are +// only fired when the undo_db is enabled. +// So we use this helper object to disable undo_db only if it is not forbidden +// with _slow_replays flag. +class auto_undo_enabler +{ + const bool _slow_replays; + undo_database& _undo_db; + bool _disabled; +public: + auto_undo_enabler(bool slow_replays, undo_database& undo_db) : + _slow_replays(slow_replays), + _undo_db(undo_db), + _disabled(false) + { + } - auto start = fc::time_point::now(); + ~auto_undo_enabler() + { + try{ + enable(); + } FC_CAPTURE_AND_LOG(("undo_db enabling crash")) + } + + void enable() + { + if(!_disabled) + return; + _undo_db.enable(); + _disabled = false; + } + + void disable() + { + if(_disabled) + return; + if(_slow_replays) + return; + _undo_db.disable(); + _disabled = true; + } +}; + +void database::reindex( fc::path data_dir ) +{ try { auto last_block = _block_id_to_block.last(); if( !last_block ) { elog( "!no last block" ); edump((last_block)); return; } + if( last_block->block_num() <= head_block_num()) return; + ilog( "reindexing blockchain" ); + auto start = fc::time_point::now(); const auto last_block_num = last_block->block_num(); + uint32_t flush_point = last_block_num < 10000 ? 0 : last_block_num - 10000; + uint32_t undo_point = last_block_num < 50 ? 0 : last_block_num - 50; - ilog( "Replaying blocks..." ); - // Right now, we leave undo_db enabled when replaying when the bookie plugin is - // enabled. It depends on new/changed/removed object notifications, and those are - // only fired when the undo_db is enabled - if (!_slow_replays) - _undo_db.disable(); - for( uint32_t i = 1; i <= last_block_num; ++i ) + ilog( "Replaying blocks, starting at ${next}...", ("next",head_block_num() + 1) ); + auto_undo_enabler undo(_slow_replays, _undo_db); + if( head_block_num() >= undo_point ) { - if( i == 1 || - i % 10000 == 0 ) - std::cerr << " " << double(i*100)/last_block_num << "% "<< i << " of " < 0 ) + _fork_db.start_block( *fetch_block_by_number( head_block_num() ) ); + } + else + { + undo.disable(); + } + for( uint32_t i = head_block_num() + 1; i <= last_block_num; ++i ) + { + if( i % 10000 == 0 ) std::cerr << " " << double(i*100)/last_block_num << "% "< block = _block_id_to_block.fetch_by_number(i); if( !block.valid() ) { @@ -94,24 +147,27 @@ void database::reindex(fc::path data_dir, const genesis_state_type& initial_allo wlog( "Dropped ${n} blocks from after the gap", ("n", dropped_count) ); break; } - if (_slow_replays) - push_block(*block, skip_fork_db | - skip_witness_signature | - skip_transaction_signatures | - skip_transaction_dupe_check | - skip_tapos_check | - skip_witness_schedule_check | - skip_authority_check); - else + if( i < undo_point && !_slow_replays) + { apply_block(*block, skip_witness_signature | skip_transaction_signatures | skip_transaction_dupe_check | skip_tapos_check | skip_witness_schedule_check | skip_authority_check); + } + else + { + undo.enable(); + push_block(*block, skip_witness_signature | + skip_transaction_signatures | + skip_transaction_dupe_check | + skip_tapos_check | + skip_witness_schedule_check | + skip_authority_check); + } } - if (!_slow_replays) - _undo_db.enable(); + undo.enable(); auto end = fc::time_point::now(); ilog( "Done reindexing, elapsed time: ${t} sec", ("t",double((end-start).count())/1000000.0 ) ); } FC_CAPTURE_AND_RETHROW( (data_dir) ) } @@ -129,10 +185,29 @@ void database::wipe(const fc::path& data_dir, bool include_blocks) void database::open( const fc::path& data_dir, - std::function genesis_loader) + std::function genesis_loader, + const std::string& db_version) { try { + bool wipe_object_db = false; + if( !fc::exists( data_dir / "db_version" ) ) + wipe_object_db = true; + else + { + std::string version_string; + fc::read_file_contents( data_dir / "db_version", version_string ); + wipe_object_db = ( version_string != db_version ); + } + if( wipe_object_db ) { + ilog("Wiping object_database due to missing or wrong version"); + object_database::wipe( data_dir ); + std::ofstream version_file( (data_dir / "db_version").generic_string().c_str(), + std::ios::out | std::ios::binary | std::ios::trunc ); + version_file.write( db_version.c_str(), db_version.size() ); + version_file.close(); + } + object_database::open(data_dir); _block_id_to_block.open(data_dir / "database" / "block_num_to_block"); @@ -140,15 +215,13 @@ void database::open( if( !find(global_property_id_type()) ) init_genesis(genesis_loader()); - fc::optional last_block = _block_id_to_block.last(); + fc::optional last_block = _block_id_to_block.last_id(); if( last_block.valid() ) { - _fork_db.start_block( *last_block ); - if( last_block->id() != head_block_id() ) - { - FC_ASSERT( head_block_num() == 0, "last block ID does not match current chain state", - ("last_block->id", last_block->id())("head_block_num",head_block_num()) ); - } + FC_ASSERT( *last_block >= head_block_id(), + "last block ID does not match current chain state", + ("last_block->id", last_block)("head_block_id",head_block_num()) ); + reindex( data_dir ); } _opened = true; } @@ -173,17 +246,9 @@ void database::close(bool rewind) while( head_block_num() > cutoff ) { - // elog("pop"); block_id_type popped_block_id = head_block_id(); pop_block(); _fork_db.remove(popped_block_id); // doesn't throw on missing - try - { - _block_id_to_block.remove(popped_block_id); - } - catch (const fc::key_not_found_exception&) - { - } } } catch ( const fc::exception& e ) diff --git a/libraries/chain/event_object.cpp b/libraries/chain/event_object.cpp index 99caf904..4c2fce14 100644 --- a/libraries/chain/event_object.cpp +++ b/libraries/chain/event_object.cpp @@ -553,30 +553,30 @@ namespace graphene { namespace chain { namespace fc { // Manually reflect event_object to variant to properly reflect "state" - void to_variant(const graphene::chain::event_object& event_obj, fc::variant& v) + void to_variant(const graphene::chain::event_object& event_obj, fc::variant& v, uint32_t max_depth) { fc::mutable_variant_object o; - o("id", event_obj.id) - ("name", event_obj.name) - ("season", event_obj.season) - ("start_time", event_obj.start_time) - ("event_group_id", event_obj.event_group_id) - ("scores", event_obj.scores) - ("status", event_obj.get_status()); + o("id", fc::variant(event_obj.id, max_depth)) + ("name", fc::variant(event_obj.name, max_depth)) + ("season", fc::variant(event_obj.season, max_depth)) + ("start_time", fc::variant(event_obj.start_time, max_depth)) + ("event_group_id", fc::variant(event_obj.event_group_id, max_depth)) + ("scores", fc::variant(event_obj.scores, max_depth)) + ("status", fc::variant(event_obj.get_status(), max_depth)); v = o; } // Manually reflect event_object to variant to properly reflect "state" - void from_variant(const fc::variant& v, graphene::chain::event_object& event_obj) + void from_variant(const fc::variant& v, graphene::chain::event_object& event_obj, uint32_t max_depth) { - event_obj.id = v["id"].as(); - event_obj.name = v["name"].as(); - event_obj.season = v["season"].as(); - event_obj.start_time = v["start_time"].as >(); - event_obj.event_group_id = v["event_group_id"].as(); - event_obj.scores = v["scores"].as>(); - graphene::chain::event_status status = v["status"].as(); + event_obj.id = v["id"].as( max_depth ); + event_obj.name = v["name"].as( max_depth ); + event_obj.season = v["season"].as( max_depth ); + event_obj.start_time = v["start_time"].as >( max_depth ); + event_obj.event_group_id = v["event_group_id"].as( max_depth ); + event_obj.scores = v["scores"].as>( max_depth ); + graphene::chain::event_status status = v["status"].as( max_depth ); const_cast(event_obj.my->state_machine.current_state())[0] = (int)status; } } //end namespace fc diff --git a/libraries/chain/game_object.cpp b/libraries/chain/game_object.cpp index 02612a77..5874fd9e 100644 --- a/libraries/chain/game_object.cpp +++ b/libraries/chain/game_object.cpp @@ -549,33 +549,33 @@ namespace graphene { namespace chain { namespace fc { // Manually reflect game_object to variant to properly reflect "state" - void to_variant(const graphene::chain::game_object& game_obj, fc::variant& v) + void to_variant(const graphene::chain::game_object& game_obj, fc::variant& v, uint32_t max_depth) { fc_elog(fc::logger::get("tournament"), "In game_obj to_variant"); elog("In game_obj to_variant"); fc::mutable_variant_object o; - o("id", game_obj.id) - ("match_id", game_obj.match_id) - ("players", game_obj.players) - ("winners", game_obj.winners) - ("game_details", game_obj.game_details) - ("next_timeout", game_obj.next_timeout) - ("state", game_obj.get_state()); + o("id", fc::variant(game_obj.id, max_depth )) + ("match_id", fc::variant(game_obj.match_id, max_depth )) + ("players", fc::variant(game_obj.players, max_depth )) + ("winners", fc::variant(game_obj.winners, max_depth )) + ("game_details", fc::variant(game_obj.game_details, max_depth )) + ("next_timeout", fc::variant(game_obj.next_timeout, max_depth )) + ("state", fc::variant(game_obj.get_state(), max_depth )); v = o; } // Manually reflect game_object to variant to properly reflect "state" - void from_variant(const fc::variant& v, graphene::chain::game_object& game_obj) + void from_variant(const fc::variant& v, graphene::chain::game_object& game_obj, uint32_t max_depth) { fc_elog(fc::logger::get("tournament"), "In game_obj from_variant"); - game_obj.id = v["id"].as(); - game_obj.match_id = v["match_id"].as(); - game_obj.players = v["players"].as >(); - game_obj.winners = v["winners"].as >(); - game_obj.game_details = v["game_details"].as(); - game_obj.next_timeout = v["next_timeout"].as >(); - graphene::chain::game_state state = v["state"].as(); + game_obj.id = v["id"].as( max_depth ); + game_obj.match_id = v["match_id"].as( max_depth ); + game_obj.players = v["players"].as >( max_depth ); + game_obj.winners = v["winners"].as >( max_depth ); + game_obj.game_details = v["game_details"].as( max_depth ); + game_obj.next_timeout = v["next_timeout"].as >( max_depth ); + graphene::chain::game_state state = v["state"].as( max_depth ); const_cast(game_obj.my->state_machine.current_state())[0] = (int)state; } } //end namespace fc diff --git a/libraries/chain/get_config.cpp b/libraries/chain/get_config.cpp index c6b35f7a..c961b950 100644 --- a/libraries/chain/get_config.cpp +++ b/libraries/chain/get_config.cpp @@ -103,11 +103,11 @@ fc::variant_object get_config() result[ "GRAPHENE_DEFAULT_WITNESS_PAY_VESTING_SECONDS" ] = GRAPHENE_DEFAULT_WITNESS_PAY_VESTING_SECONDS; result[ "GRAPHENE_DEFAULT_WORKER_BUDGET_PER_DAY" ] = GRAPHENE_DEFAULT_WORKER_BUDGET_PER_DAY; result[ "GRAPHENE_MAX_INTEREST_APR" ] = GRAPHENE_MAX_INTEREST_APR; - result[ "GRAPHENE_COMMITTEE_ACCOUNT" ] = GRAPHENE_COMMITTEE_ACCOUNT; - result[ "GRAPHENE_WITNESS_ACCOUNT" ] = GRAPHENE_WITNESS_ACCOUNT; - result[ "GRAPHENE_RELAXED_COMMITTEE_ACCOUNT" ] = GRAPHENE_RELAXED_COMMITTEE_ACCOUNT; - result[ "GRAPHENE_NULL_ACCOUNT" ] = GRAPHENE_NULL_ACCOUNT; - result[ "GRAPHENE_TEMP_ACCOUNT" ] = GRAPHENE_TEMP_ACCOUNT; + result[ "GRAPHENE_COMMITTEE_ACCOUNT" ] = fc::variant(GRAPHENE_COMMITTEE_ACCOUNT, GRAPHENE_MAX_NESTED_OBJECTS); + result[ "GRAPHENE_WITNESS_ACCOUNT" ] = fc::variant(GRAPHENE_WITNESS_ACCOUNT, GRAPHENE_MAX_NESTED_OBJECTS); + result[ "GRAPHENE_RELAXED_COMMITTEE_ACCOUNT" ] = fc::variant(GRAPHENE_RELAXED_COMMITTEE_ACCOUNT, GRAPHENE_MAX_NESTED_OBJECTS); + result[ "GRAPHENE_NULL_ACCOUNT" ] = fc::variant(GRAPHENE_NULL_ACCOUNT, GRAPHENE_MAX_NESTED_OBJECTS); + result[ "GRAPHENE_TEMP_ACCOUNT" ] = fc::variant(GRAPHENE_TEMP_ACCOUNT, GRAPHENE_MAX_NESTED_OBJECTS); return result; } diff --git a/libraries/chain/include/graphene/chain/account_object.hpp b/libraries/chain/include/graphene/chain/account_object.hpp index 914ade33..4e940326 100644 --- a/libraries/chain/include/graphene/chain/account_object.hpp +++ b/libraries/chain/include/graphene/chain/account_object.hpp @@ -278,6 +278,25 @@ namespace graphene { namespace chain { */ class account_member_index : public secondary_index { + /* std::less::operator() is less efficient so using key_compare here. + * Let assume that it has two keys key1 and key2 and we want to insert key3. + * the insert function needs to first call std::less::operator() with key1 and key3. + * Assume std::less::operator()(key1, key3) returns false. + * It has to call std::less::operator() again with the keys switched, + * std::less::operator()(key3, key1), to decide whether key1 is equal to key3 or + * key3 is greater than key1. There are two calls to std::less::operator() to make + * a decision if the first call returns false. + * std::map::insert and std::set used key_compare, + * there would be sufficient information to make the right decision using just one call. + */ + class key_compare { + public: + inline bool operator()( const public_key_type& a, const public_key_type& b )const + { + return a.key_data < b.key_data; + } + }; + public: virtual void object_inserted( const object& obj ) override; virtual void object_removed( const object& obj ) override; @@ -287,18 +306,18 @@ namespace graphene { namespace chain { /** given an account or key, map it to the set of accounts that reference it in an active or owner authority */ map< account_id_type, set > account_to_account_memberships; - map< public_key_type, set > account_to_key_memberships; + map< public_key_type, set, key_compare > account_to_key_memberships; /** some accounts use address authorities in the genesis block */ map< address, set > account_to_address_memberships; protected: set get_account_members( const account_object& a )const; - set get_key_members( const account_object& a )const; + set get_key_members( const account_object& a )const; set
get_address_members( const account_object& a )const; set before_account_members; - set before_key_members; + set before_key_members; set
before_address_members; }; @@ -344,7 +363,30 @@ namespace graphene { namespace chain { }; - struct by_account_asset; + /** + * @brief This secondary index will allow fast access to the balance objects + * that belonging to an account. + */ + class balances_by_account_index : public secondary_index + { + public: + virtual void object_inserted( const object& obj ) override; + virtual void object_removed( const object& obj ) override; + virtual void about_to_modify( const object& before ) override; + virtual void object_modified( const object& after ) override; + + const map< asset_id_type, const account_balance_object* >& get_account_balances( const account_id_type& acct )const; + const account_balance_object* get_account_balance( const account_id_type& acct, const asset_id_type& asset )const; + + private: + static const uint8_t bits; + static const uint64_t mask; + + /** Maps each account to its balance objects */ + vector< vector< map< asset_id_type, const account_balance_object* > > > balances; + std::stack< object_id_type > ids_being_modified; + }; + struct by_asset_balance; /** * @ingroup object_index @@ -353,13 +395,6 @@ namespace graphene { namespace chain { account_balance_object, indexed_by< ordered_unique< tag, member< object, object_id_type, &object::id > >, - ordered_unique< tag, - composite_key< - account_balance_object, - member, - member - > - >, ordered_unique< tag, composite_key< account_balance_object, @@ -399,6 +434,26 @@ namespace graphene { namespace chain { */ typedef generic_index account_index; + struct by_owner; + struct by_maintenance_seq; + + /** + * @ingroup object_index + */ + typedef multi_index_container< + account_statistics_object, + indexed_by< + ordered_unique< tag, member< object, object_id_type, &object::id > >, + ordered_unique< tag, + member< account_statistics_object, account_id_type, &account_statistics_object::owner > > + > + > account_stats_multi_index_type; + + /** + * @ingroup object_index + */ + typedef generic_index account_stats_index; + struct by_dividend_payout_account{}; // use when calculating pending payouts struct by_dividend_account_payout{}; // use when doing actual payouts struct by_account_dividend_payout{}; // use in get_full_accounts() diff --git a/libraries/chain/include/graphene/chain/asset_object.hpp b/libraries/chain/include/graphene/chain/asset_object.hpp index afd5215a..f1df4681 100644 --- a/libraries/chain/include/graphene/chain/asset_object.hpp +++ b/libraries/chain/include/graphene/chain/asset_object.hpp @@ -230,8 +230,16 @@ namespace graphene { namespace chain { share_type settlement_fund; ///@} + /// The time when @ref current_feed would expire time_point_sec feed_expiration_time()const - { return current_feed_publication_time + options.feed_lifetime_sec; } + { + uint32_t current_feed_seconds = current_feed_publication_time.sec_since_epoch(); + if( std::numeric_limits::max() - current_feed_seconds <= options.feed_lifetime_sec ) + return time_point_sec::maximum(); + else + return current_feed_publication_time + options.feed_lifetime_sec; + } + bool feed_is_expired_before_hardfork_615(time_point_sec current_time)const { return feed_expiration_time() >= current_time; } bool feed_is_expired(time_point_sec current_time)const diff --git a/libraries/chain/include/graphene/chain/betting_market_object.hpp b/libraries/chain/include/graphene/chain/betting_market_object.hpp index cb815739..36e8e664 100644 --- a/libraries/chain/include/graphene/chain/betting_market_object.hpp +++ b/libraries/chain/include/graphene/chain/betting_market_object.hpp @@ -37,10 +37,10 @@ namespace graphene { namespace chain { } } namespace fc { - void to_variant(const graphene::chain::betting_market_object& betting_market_obj, fc::variant& v); - void from_variant(const fc::variant& v, graphene::chain::betting_market_object& betting_market_obj); - void to_variant(const graphene::chain::betting_market_group_object& betting_market_group_obj, fc::variant& v); - void from_variant(const fc::variant& v, graphene::chain::betting_market_group_object& betting_market_group_obj); + void to_variant(const graphene::chain::betting_market_object& betting_market_obj, fc::variant& v, uint32_t max_depth = 1); + void from_variant(const fc::variant& v, graphene::chain::betting_market_object& betting_market_obj, uint32_t max_depth = 1); + void to_variant(const graphene::chain::betting_market_group_object& betting_market_group_obj, fc::variant& v, uint32_t max_depth = 1); + void from_variant(const fc::variant& v, graphene::chain::betting_market_group_object& betting_market_group_obj, uint32_t max_depth = 1); } //end namespace fc namespace graphene { namespace chain { @@ -110,8 +110,8 @@ class betting_market_group_object : public graphene::db::abstract_object< bettin template friend Stream& operator>>( Stream& s, betting_market_group_object& betting_market_group_obj ); - friend void ::fc::to_variant(const graphene::chain::betting_market_group_object& betting_market_group_obj, fc::variant& v); - friend void ::fc::from_variant(const fc::variant& v, graphene::chain::betting_market_group_object& betting_market_group_obj); + friend void ::fc::to_variant(const graphene::chain::betting_market_group_object& betting_market_group_obj, fc::variant& v, uint32_t max_depth); + friend void ::fc::from_variant(const fc::variant& v, graphene::chain::betting_market_group_object& betting_market_group_obj, uint32_t max_depth); void pack_impl(std::ostream& stream) const; void unpack_impl(std::istream& stream); @@ -166,8 +166,8 @@ class betting_market_object : public graphene::db::abstract_object< betting_mark template friend Stream& operator>>( Stream& s, betting_market_object& betting_market_obj ); - friend void ::fc::to_variant(const graphene::chain::betting_market_object& betting_market_obj, fc::variant& v); - friend void ::fc::from_variant(const fc::variant& v, graphene::chain::betting_market_object& betting_market_obj); + friend void ::fc::to_variant(const graphene::chain::betting_market_object& betting_market_obj, fc::variant& v, uint32_t max_depth); + friend void ::fc::from_variant(const fc::variant& v, graphene::chain::betting_market_object& betting_market_obj, uint32_t max_depth); void pack_impl(std::ostream& stream) const; void unpack_impl(std::istream& stream); diff --git a/libraries/chain/include/graphene/chain/block_database.hpp b/libraries/chain/include/graphene/chain/block_database.hpp index d1f613c1..d902cd1b 100644 --- a/libraries/chain/include/graphene/chain/block_database.hpp +++ b/libraries/chain/include/graphene/chain/block_database.hpp @@ -26,6 +26,8 @@ #include namespace graphene { namespace chain { + class index_entry; + class block_database { public: @@ -44,6 +46,8 @@ namespace graphene { namespace chain { optional last()const; optional last_id()const; private: + optional last_index_entry()const; + fc::path _index_filename; mutable std::fstream _blocks; mutable std::fstream _block_num_to_pos; }; diff --git a/libraries/chain/include/graphene/chain/config.hpp b/libraries/chain/include/graphene/chain/config.hpp index becf73d6..a5354f85 100644 --- a/libraries/chain/include/graphene/chain/config.hpp +++ b/libraries/chain/include/graphene/chain/config.hpp @@ -211,6 +211,7 @@ { 10000000, 100000} } /* <= 1000: 10.00 */ #define GRAPHENE_DEFAULT_BETTING_PERCENT_FEE (2 * GRAPHENE_1_PERCENT) #define GRAPHENE_DEFAULT_LIVE_BETTING_DELAY_TIME 5 // seconds +#define GRAPHENE_MAX_NESTED_OBJECTS (200) #define TOURNAMENT_MIN_ROUND_DELAY 0 #define TOURNAMENT_MAX_ROUND_DELAY 600 #define TOURNAMENT_MIN_TIME_PER_COMMIT_MOVE 0 diff --git a/libraries/chain/include/graphene/chain/database.hpp b/libraries/chain/include/graphene/chain/database.hpp index b513fd2f..483bfc8a 100644 --- a/libraries/chain/include/graphene/chain/database.hpp +++ b/libraries/chain/include/graphene/chain/database.hpp @@ -91,10 +91,12 @@ namespace graphene { namespace chain { * * @param data_dir Path to open or create database in * @param genesis_loader A callable object which returns the genesis state to initialize new databases on + * @param db_version a version string that changes when the internal database format and/or logic is modified */ void open( const fc::path& data_dir, - std::function genesis_loader ); + std::function genesis_loader, + const std::string& db_version ); /** * @brief Rebuild object graph from block history and open detabase @@ -102,7 +104,7 @@ namespace graphene { namespace chain { * This method may be called after or instead of @ref database::open, and will rebuild the object graph by * replaying blockchain history. When this method exits successfully, the database will be open. */ - void reindex(fc::path data_dir, const genesis_state_type& initial_allocation = genesis_state_type()); + void reindex(fc::path data_dir); /** * @brief wipe Delete database from disk, and potentially the raw chain as well. diff --git a/libraries/chain/include/graphene/chain/event_object.hpp b/libraries/chain/include/graphene/chain/event_object.hpp index c8c9b493..4258cbb1 100644 --- a/libraries/chain/include/graphene/chain/event_object.hpp +++ b/libraries/chain/include/graphene/chain/event_object.hpp @@ -36,8 +36,8 @@ namespace graphene { namespace chain { } } namespace fc { - void to_variant(const graphene::chain::event_object& event_obj, fc::variant& v); - void from_variant(const fc::variant& v, graphene::chain::event_object& event_obj); + void to_variant(const graphene::chain::event_object& event_obj, fc::variant& v, uint32_t max_depth = 1); + void from_variant(const fc::variant& v, graphene::chain::event_object& event_obj, uint32_t max_depth = 1); } //end namespace fc namespace graphene { namespace chain { @@ -77,8 +77,8 @@ class event_object : public graphene::db::abstract_object< event_object > template friend Stream& operator>>( Stream& s, event_object& event_obj ); - friend void ::fc::to_variant(const graphene::chain::event_object& event_obj, fc::variant& v); - friend void ::fc::from_variant(const fc::variant& v, graphene::chain::event_object& event_obj); + friend void ::fc::to_variant(const graphene::chain::event_object& event_obj, fc::variant& v, uint32_t max_depth); + friend void ::fc::from_variant(const fc::variant& v, graphene::chain::event_object& event_obj, uint32_t max_depth); void pack_impl(std::ostream& stream) const; void unpack_impl(std::istream& stream); diff --git a/libraries/chain/include/graphene/chain/game_object.hpp b/libraries/chain/include/graphene/chain/game_object.hpp index eff04023..abef1444 100644 --- a/libraries/chain/include/graphene/chain/game_object.hpp +++ b/libraries/chain/include/graphene/chain/game_object.hpp @@ -36,8 +36,8 @@ namespace graphene { namespace chain { } } namespace fc { - void to_variant(const graphene::chain::game_object& game_obj, fc::variant& v); - void from_variant(const fc::variant& v, graphene::chain::game_object& game_obj); + void to_variant(const graphene::chain::game_object& game_obj, fc::variant& v, uint32_t max_depth = 1); + void from_variant(const fc::variant& v, graphene::chain::game_object& game_obj, uint32_t max_depth = 1); } //end namespace fc namespace graphene { namespace chain { @@ -92,8 +92,8 @@ namespace graphene { namespace chain { template friend Stream& operator>>( Stream& s, game_object& game_obj ); - friend void ::fc::to_variant(const graphene::chain::game_object& game_obj, fc::variant& v); - friend void ::fc::from_variant(const fc::variant& v, graphene::chain::game_object& game_obj); + friend void ::fc::to_variant(const graphene::chain::game_object& game_obj, fc::variant& v, uint32_t max_depth); + friend void ::fc::from_variant(const fc::variant& v, graphene::chain::game_object& game_obj, uint32_t max_depth); void pack_impl(std::ostream& stream) const; void unpack_impl(std::istream& stream); diff --git a/libraries/chain/include/graphene/chain/match_object.hpp b/libraries/chain/include/graphene/chain/match_object.hpp index 67146e38..72c346a7 100644 --- a/libraries/chain/include/graphene/chain/match_object.hpp +++ b/libraries/chain/include/graphene/chain/match_object.hpp @@ -12,8 +12,8 @@ namespace graphene { namespace chain { } } namespace fc { - void to_variant(const graphene::chain::match_object& match_obj, fc::variant& v); - void from_variant(const fc::variant& v, graphene::chain::match_object& match_obj); + void to_variant(const graphene::chain::match_object& match_obj, fc::variant& v, uint32_t max_depth = 1); + void from_variant(const fc::variant& v, graphene::chain::match_object& match_obj, uint32_t max_depth = 1); } //end namespace fc namespace graphene { namespace chain { @@ -84,8 +84,8 @@ namespace graphene { namespace chain { template friend Stream& operator>>( Stream& s, match_object& match_obj ); - friend void ::fc::to_variant(const graphene::chain::match_object& match_obj, fc::variant& v); - friend void ::fc::from_variant(const fc::variant& v, graphene::chain::match_object& match_obj); + friend void ::fc::to_variant(const graphene::chain::match_object& match_obj, fc::variant& v, uint32_t max_depth); + friend void ::fc::from_variant(const fc::variant& v, graphene::chain::match_object& match_obj, uint32_t max_depth); void pack_impl(std::ostream& stream) const; void unpack_impl(std::istream& stream); diff --git a/libraries/chain/include/graphene/chain/proposal_object.hpp b/libraries/chain/include/graphene/chain/proposal_object.hpp index 0affcbab..56f44749 100644 --- a/libraries/chain/include/graphene/chain/proposal_object.hpp +++ b/libraries/chain/include/graphene/chain/proposal_object.hpp @@ -54,6 +54,7 @@ class proposal_object : public abstract_object std::string fail_reason; bool is_authorized_to_execute(database& db) const; + }; /** @@ -95,4 +96,4 @@ typedef generic_index proposal_ FC_REFLECT_DERIVED( graphene::chain::proposal_object, (graphene::chain::object), (expiration_time)(review_period_time)(proposed_transaction)(required_active_approvals) (available_active_approvals)(required_owner_approvals)(available_owner_approvals) - (available_key_approvals)(proposer) ) + (available_key_approvals)(proposer)(fail_reason)) diff --git a/libraries/chain/include/graphene/chain/protocol/address.hpp b/libraries/chain/include/graphene/chain/protocol/address.hpp index 00331c08..b225b42c 100644 --- a/libraries/chain/include/graphene/chain/protocol/address.hpp +++ b/libraries/chain/include/graphene/chain/protocol/address.hpp @@ -78,8 +78,8 @@ namespace graphene { namespace chain { namespace fc { - void to_variant( const graphene::chain::address& var, fc::variant& vo ); - void from_variant( const fc::variant& var, graphene::chain::address& vo ); + void to_variant( const graphene::chain::address& var, fc::variant& vo, uint32_t max_depth = 1 ); + void from_variant( const fc::variant& var, graphene::chain::address& vo, uint32_t max_depth = 1 ); } namespace std diff --git a/libraries/chain/include/graphene/chain/protocol/ext.hpp b/libraries/chain/include/graphene/chain/protocol/ext.hpp index ac775535..31f66506 100644 --- a/libraries/chain/include/graphene/chain/protocol/ext.hpp +++ b/libraries/chain/include/graphene/chain/protocol/ext.hpp @@ -145,9 +145,10 @@ namespace fc { template< typename T > struct graphene_extension_from_variant_visitor { - graphene_extension_from_variant_visitor( const variant_object& v, T& val ) - : vo( v ), value( val ) + graphene_extension_from_variant_visitor( const variant_object& v, T& val, uint32_t max_depth ) + : vo( v ), value( val ), _max_depth(max_depth - 1) { + FC_ASSERT( max_depth > 0, "Recursion depth exceeded!" ); count_left = vo.size(); } @@ -157,7 +158,7 @@ struct graphene_extension_from_variant_visitor auto it = vo.find(name); if( it != vo.end() ) { - from_variant( it->value(), (value.*member) ); + from_variant( it->value(), (value.*member), _max_depth ); assert( count_left > 0 ); // x.find(k) returns true for n distinct values of k only if x.size() >= n --count_left; } @@ -165,11 +166,12 @@ struct graphene_extension_from_variant_visitor const variant_object& vo; T& value; + const uint32_t _max_depth; mutable uint32_t count_left = 0; }; template< typename T > -void from_variant( const fc::variant& var, graphene::chain::extension& value ) +void from_variant( const fc::variant& var, graphene::chain::extension& value, uint32_t max_depth ) { value = graphene::chain::extension(); if( var.is_null() ) @@ -180,7 +182,7 @@ void from_variant( const fc::variant& var, graphene::chain::extension& value return; } - graphene_extension_from_variant_visitor vtor( var.get_object(), value.value ); + graphene_extension_from_variant_visitor vtor( var.get_object(), value.value, max_depth ); fc::reflector::visit( vtor ); FC_ASSERT( vtor.count_left == 0 ); // unrecognized extension throws here } @@ -188,23 +190,23 @@ void from_variant( const fc::variant& var, graphene::chain::extension& value template< typename T > struct graphene_extension_to_variant_visitor { - graphene_extension_to_variant_visitor( const T& v ) : value(v) {} + graphene_extension_to_variant_visitor( const T& v, uint32_t max_depth ) : value(v), mvo(max_depth) {} template void operator()( const char* name )const { if( (value.*member).valid() ) - mvo[ name ] = (value.*member); + mvo( name, value.*member ); } const T& value; - mutable mutable_variant_object mvo; + mutable limited_mutable_variant_object mvo; }; template< typename T > -void to_variant( const graphene::chain::extension& value, fc::variant& var ) +void to_variant( const graphene::chain::extension& value, fc::variant& var, uint32_t max_depth ) { - graphene_extension_to_variant_visitor vtor( value.value ); + graphene_extension_to_variant_visitor vtor( value.value, max_depth ); fc::reflector::visit( vtor ); var = vtor.mvo; } diff --git a/libraries/chain/include/graphene/chain/protocol/transaction.hpp b/libraries/chain/include/graphene/chain/protocol/transaction.hpp index 4d529a27..95c39961 100644 --- a/libraries/chain/include/graphene/chain/protocol/transaction.hpp +++ b/libraries/chain/include/graphene/chain/protocol/transaction.hpp @@ -165,12 +165,30 @@ namespace graphene { namespace chain { uint32_t max_recursion = GRAPHENE_MAX_SIG_CHECK_DEPTH ) const; - flat_set get_signature_keys( const chain_id_type& chain_id )const; + /** + * @brief Extract public keys from signatures with given chain ID. + * @param chain_id A chain ID + * @return Public keys + * @note If @ref signees is empty, E.G. when it's the first time calling + * this function for the signed transaction, public keys will be + * extracted with given chain ID, and be stored into the mutable + * @ref signees field, then @ref signees will be returned; + * otherwise, the @ref chain_id parameter will be ignored, and + * @ref signees will be returned directly. + */ + const flat_set& get_signature_keys( const chain_id_type& chain_id )const; + /** Signatures */ vector signatures; - /// Removes all operations and signatures - void clear() { operations.clear(); signatures.clear(); } + /** Public keys extracted from signatures */ + mutable flat_set signees; + + /// Removes all operations, signatures and signees + void clear() { operations.clear(); signatures.clear(); signees.clear(); } + + /// Removes all signatures and signees + void clear_signatures() { signatures.clear(); signees.clear(); } }; void verify_authority( const vector& ops, const flat_set& sigs, @@ -209,5 +227,6 @@ namespace graphene { namespace chain { } } // graphene::chain FC_REFLECT( graphene::chain::transaction, (ref_block_num)(ref_block_prefix)(expiration)(operations)(extensions) ) +// Note: not reflecting signees field for backward compatibility; in addition, it should not be in p2p messages FC_REFLECT_DERIVED( graphene::chain::signed_transaction, (graphene::chain::transaction), (signatures) ) FC_REFLECT_DERIVED( graphene::chain::processed_transaction, (graphene::chain::signed_transaction), (operation_results) ) diff --git a/libraries/chain/include/graphene/chain/protocol/types.hpp b/libraries/chain/include/graphene/chain/protocol/types.hpp index 4df38372..c2c92ca3 100644 --- a/libraries/chain/include/graphene/chain/protocol/types.hpp +++ b/libraries/chain/include/graphene/chain/protocol/types.hpp @@ -367,12 +367,12 @@ namespace graphene { namespace chain { namespace fc { - void to_variant( const graphene::chain::public_key_type& var, fc::variant& vo ); - void from_variant( const fc::variant& var, graphene::chain::public_key_type& vo ); - void to_variant( const graphene::chain::extended_public_key_type& var, fc::variant& vo ); - void from_variant( const fc::variant& var, graphene::chain::extended_public_key_type& vo ); - void to_variant( const graphene::chain::extended_private_key_type& var, fc::variant& vo ); - void from_variant( const fc::variant& var, graphene::chain::extended_private_key_type& vo ); + void to_variant( const graphene::chain::public_key_type& var, fc::variant& vo, uint32_t max_depth = 2 ); + void from_variant( const fc::variant& var, graphene::chain::public_key_type& vo, uint32_t max_depth = 2 ); + void to_variant( const graphene::chain::extended_public_key_type& var, fc::variant& vo, uint32_t max_depth = 2 ); + void from_variant( const fc::variant& var, graphene::chain::extended_public_key_type& vo, uint32_t max_depth = 2 ); + void to_variant( const graphene::chain::extended_private_key_type& var, fc::variant& vo, uint32_t max_depth = 2 ); + void from_variant( const fc::variant& var, graphene::chain::extended_private_key_type& vo, uint32_t max_depth = 2 ); } FC_REFLECT( graphene::chain::public_key_type, (key_data) ) diff --git a/libraries/chain/include/graphene/chain/protocol/vote.hpp b/libraries/chain/include/graphene/chain/protocol/vote.hpp index 215d4902..67536f7a 100644 --- a/libraries/chain/include/graphene/chain/protocol/vote.hpp +++ b/libraries/chain/include/graphene/chain/protocol/vote.hpp @@ -141,8 +141,8 @@ namespace fc class variant; -void to_variant( const graphene::chain::vote_id_type& var, fc::variant& vo ); -void from_variant( const fc::variant& var, graphene::chain::vote_id_type& vo ); +void to_variant( const graphene::chain::vote_id_type& var, fc::variant& vo, uint32_t max_depth = 1 ); +void from_variant( const fc::variant& var, graphene::chain::vote_id_type& vo, uint32_t max_depth = 1 ); } // fc diff --git a/libraries/chain/include/graphene/chain/pts_address.hpp b/libraries/chain/include/graphene/chain/pts_address.hpp index 8c53fb2e..636e2f11 100644 --- a/libraries/chain/include/graphene/chain/pts_address.hpp +++ b/libraries/chain/include/graphene/chain/pts_address.hpp @@ -73,6 +73,6 @@ FC_REFLECT( graphene::chain::pts_address, (addr) ) namespace fc { - void to_variant( const graphene::chain::pts_address& var, fc::variant& vo ); - void from_variant( const fc::variant& var, graphene::chain::pts_address& vo ); + void to_variant( const graphene::chain::pts_address& var, fc::variant& vo, uint32_t max_depth = 1 ); + void from_variant( const fc::variant& var, graphene::chain::pts_address& vo, uint32_t max_depth = 1 ); } diff --git a/libraries/chain/include/graphene/chain/tournament_object.hpp b/libraries/chain/include/graphene/chain/tournament_object.hpp index ffde72f8..140770e2 100644 --- a/libraries/chain/include/graphene/chain/tournament_object.hpp +++ b/libraries/chain/include/graphene/chain/tournament_object.hpp @@ -12,8 +12,8 @@ namespace graphene { namespace chain { } } namespace fc { - void to_variant(const graphene::chain::tournament_object& tournament_obj, fc::variant& v); - void from_variant(const fc::variant& v, graphene::chain::tournament_object& tournament_obj); + void to_variant(const graphene::chain::tournament_object& tournament_obj, fc::variant& v, uint32_t max_depth = 1); + void from_variant(const fc::variant& v, graphene::chain::tournament_object& tournament_obj, uint32_t max_depth = 1); } //end namespace fc namespace graphene { namespace chain { @@ -108,8 +108,8 @@ namespace graphene { namespace chain { template friend Stream& operator>>( Stream& s, tournament_object& tournament_obj ); - friend void ::fc::to_variant(const graphene::chain::tournament_object& tournament_obj, fc::variant& v); - friend void ::fc::from_variant(const fc::variant& v, graphene::chain::tournament_object& tournament_obj); + friend void ::fc::to_variant(const graphene::chain::tournament_object& tournament_obj, fc::variant& v, uint32_t max_depth); + friend void ::fc::from_variant(const fc::variant& v, graphene::chain::tournament_object& tournament_obj, uint32_t max_depth); void pack_impl(std::ostream& stream) const; void unpack_impl(std::istream& stream); diff --git a/libraries/chain/match_object.cpp b/libraries/chain/match_object.cpp index 7819d21e..e11f0e8a 100644 --- a/libraries/chain/match_object.cpp +++ b/libraries/chain/match_object.cpp @@ -364,41 +364,41 @@ namespace graphene { namespace chain { namespace fc { // Manually reflect match_object to variant to properly reflect "state" - void to_variant(const graphene::chain::match_object& match_obj, fc::variant& v) + void to_variant(const graphene::chain::match_object& match_obj, fc::variant& v, uint32_t max_depth) { try { fc_elog(fc::logger::get("tournament"), "In match_obj to_variant"); elog("In match_obj to_variant"); fc::mutable_variant_object o; - o("id", match_obj.id) - ("tournament_id", match_obj.tournament_id) - ("players", match_obj.players) - ("games", match_obj.games) - ("game_winners", match_obj.game_winners) - ("number_of_wins", match_obj.number_of_wins) - ("number_of_ties", match_obj.number_of_ties) - ("match_winners", match_obj.match_winners) - ("start_time", match_obj.start_time) - ("end_time", match_obj.end_time) - ("state", match_obj.get_state()); + o("id", fc::variant(match_obj.id, max_depth)) + ("tournament_id", fc::variant(match_obj.tournament_id, max_depth)) + ("players", fc::variant(match_obj.players, max_depth)) + ("games", fc::variant(match_obj.games, max_depth)) + ("game_winners", fc::variant(match_obj.game_winners, max_depth)) + ("number_of_wins", fc::variant(match_obj.number_of_wins, max_depth)) + ("number_of_ties", fc::variant(match_obj.number_of_ties, max_depth)) + ("match_winners", fc::variant(match_obj.match_winners, max_depth)) + ("start_time", fc::variant(match_obj.start_time, max_depth)) + ("end_time", fc::variant(match_obj.end_time, max_depth)) + ("state", fc::variant(match_obj.get_state(), max_depth)); v = o; } FC_RETHROW_EXCEPTIONS(warn, "") } // Manually reflect match_object to variant to properly reflect "state" - void from_variant(const fc::variant& v, graphene::chain::match_object& match_obj) + void from_variant(const fc::variant& v, graphene::chain::match_object& match_obj, uint32_t max_depth) { try { fc_elog(fc::logger::get("tournament"), "In match_obj from_variant"); - match_obj.id = v["id"].as(); - match_obj.tournament_id = v["tournament_id"].as(); - match_obj.players = v["players"].as >(); - match_obj.games = v["games"].as >(); - match_obj.game_winners = v["game_winners"].as > >(); - match_obj.number_of_wins = v["number_of_wins"].as >(); - match_obj.number_of_ties = v["number_of_ties"].as(); - match_obj.match_winners = v["match_winners"].as >(); - match_obj.start_time = v["start_time"].as(); - match_obj.end_time = v["end_time"].as >(); - graphene::chain::match_state state = v["state"].as(); + match_obj.id = v["id"].as( max_depth ); + match_obj.tournament_id = v["tournament_id"].as( max_depth ); + match_obj.players = v["players"].as >( max_depth ); + match_obj.games = v["games"].as >( max_depth ); + match_obj.game_winners = v["game_winners"].as > >( max_depth ); + match_obj.number_of_wins = v["number_of_wins"].as >( max_depth ); + match_obj.number_of_ties = v["number_of_ties"].as( max_depth ); + match_obj.match_winners = v["match_winners"].as >( max_depth ); + match_obj.start_time = v["start_time"].as( max_depth ); + match_obj.end_time = v["end_time"].as >( max_depth ); + graphene::chain::match_state state = v["state"].as( max_depth ); const_cast(match_obj.my->state_machine.current_state())[0] = (int)state; } FC_RETHROW_EXCEPTIONS(warn, "") } } //end namespace fc diff --git a/libraries/chain/protocol/address.cpp b/libraries/chain/protocol/address.cpp index 42e03cc2..19bb4df5 100644 --- a/libraries/chain/protocol/address.cpp +++ b/libraries/chain/protocol/address.cpp @@ -101,11 +101,11 @@ namespace graphene { namespace fc { - void to_variant( const graphene::chain::address& var, variant& vo ) + void to_variant( const graphene::chain::address& var, variant& vo, uint32_t max_depth ) { vo = std::string(var); } - void from_variant( const variant& var, graphene::chain::address& vo ) + void from_variant( const variant& var, graphene::chain::address& vo, uint32_t max_depth ) { vo = graphene::chain::address( var.as_string() ); } diff --git a/libraries/chain/protocol/transaction.cpp b/libraries/chain/protocol/transaction.cpp index 5faf1c0a..a11e3335 100644 --- a/libraries/chain/protocol/transaction.cpp +++ b/libraries/chain/protocol/transaction.cpp @@ -71,6 +71,7 @@ const signature_type& graphene::chain::signed_transaction::sign(const private_ke { digest_type h = sig_digest( chain_id ); signatures.push_back(key.sign_compact(h)); + signees.clear(); // Clear signees since it may be inconsistent after added a new signature return signatures.back(); } @@ -297,22 +298,27 @@ void verify_authority( const vector& ops, const flat_set signed_transaction::get_signature_keys( const chain_id_type& chain_id )const +const flat_set& signed_transaction::get_signature_keys( const chain_id_type& chain_id )const { try { - auto d = sig_digest( chain_id ); - flat_set result; - for( const auto& sig : signatures ) + // Strictly we should check whether the given chain ID is same as the one used to initialize the `signees` field. + // However, we don't pass in another chain ID so far, for better performance, we skip the check. + if( signees.empty() && !signatures.empty() ) { - GRAPHENE_ASSERT( - result.insert( fc::ecc::public_key(sig,d) ).second, - tx_duplicate_sig, - "Duplicate Signature detected" ); + auto d = sig_digest( chain_id ); + flat_set result; + for( const auto& sig : signatures ) + { + GRAPHENE_ASSERT( + result.insert( fc::ecc::public_key(sig,d) ).second, + tx_duplicate_sig, + "Duplicate Signature detected" ); + } + signees = std::move( result ); } - return result; + return signees; } FC_CAPTURE_AND_RETHROW() } - set signed_transaction::get_required_signatures( const chain_id_type& chain_id, const flat_set& available_keys, @@ -325,8 +331,8 @@ set signed_transaction::get_required_signatures( vector other; get_required_authorities( required_active, required_owner, other ); - - sign_state s(get_signature_keys( chain_id ),get_active,available_keys); + const flat_set& signature_keys = get_signature_keys( chain_id ); + sign_state s( signature_keys, get_active, available_keys ); s.max_recursion = max_recursion_depth; for( const auto& auth : other ) diff --git a/libraries/chain/protocol/types.cpp b/libraries/chain/protocol/types.cpp index 6e3bf1fb..b7cac207 100644 --- a/libraries/chain/protocol/types.cpp +++ b/libraries/chain/protocol/types.cpp @@ -248,32 +248,32 @@ namespace graphene { namespace chain { namespace fc { using namespace std; - void to_variant( const graphene::chain::public_key_type& var, fc::variant& vo ) + void to_variant( const graphene::chain::public_key_type& var, fc::variant& vo, uint32_t max_depth ) { vo = std::string( var ); } - void from_variant( const fc::variant& var, graphene::chain::public_key_type& vo ) + void from_variant( const fc::variant& var, graphene::chain::public_key_type& vo, uint32_t max_depth ) { vo = graphene::chain::public_key_type( var.as_string() ); } - void to_variant( const graphene::chain::extended_public_key_type& var, fc::variant& vo ) + void to_variant( const graphene::chain::extended_public_key_type& var, fc::variant& vo, uint32_t max_depth ) { vo = std::string( var ); } - void from_variant( const fc::variant& var, graphene::chain::extended_public_key_type& vo ) + void from_variant( const fc::variant& var, graphene::chain::extended_public_key_type& vo, uint32_t max_depth ) { vo = graphene::chain::extended_public_key_type( var.as_string() ); } - void to_variant( const graphene::chain::extended_private_key_type& var, fc::variant& vo ) + void to_variant( const graphene::chain::extended_private_key_type& var, fc::variant& vo, uint32_t max_depth ) { vo = std::string( var ); } - void from_variant( const fc::variant& var, graphene::chain::extended_private_key_type& vo ) + void from_variant( const fc::variant& var, graphene::chain::extended_private_key_type& vo, uint32_t max_depth ) { vo = graphene::chain::extended_private_key_type( var.as_string() ); } diff --git a/libraries/chain/protocol/vote.cpp b/libraries/chain/protocol/vote.cpp index 44be9bca..f78f2b4f 100644 --- a/libraries/chain/protocol/vote.cpp +++ b/libraries/chain/protocol/vote.cpp @@ -38,12 +38,12 @@ vote_id_type get_next_vote_id( global_property_object& gpo, vote_id_type::vote_t namespace fc { -void to_variant(const graphene::chain::vote_id_type& var, variant& vo) +void to_variant( const graphene::chain::vote_id_type& var, variant& vo, uint32_t max_depth ) { vo = string(var); } -void from_variant(const variant& var, graphene::chain::vote_id_type& vo) +void from_variant( const variant& var, graphene::chain::vote_id_type& vo, uint32_t max_depth ) { vo = graphene::chain::vote_id_type(var.as_string()); } diff --git a/libraries/chain/pts_address.cpp b/libraries/chain/pts_address.cpp index d2b8c33c..27f3d256 100644 --- a/libraries/chain/pts_address.cpp +++ b/libraries/chain/pts_address.cpp @@ -89,11 +89,11 @@ namespace graphene { namespace chain { namespace fc { - void to_variant( const graphene::chain::pts_address& var, variant& vo ) + void to_variant( const graphene::chain::pts_address& var, variant& vo, uint32_t max_depth ) { vo = std::string(var); } - void from_variant( const variant& var, graphene::chain::pts_address& vo ) + void from_variant( const variant& var, graphene::chain::pts_address& vo, uint32_t max_depth ) { vo = graphene::chain::pts_address( var.as_string() ); } diff --git a/libraries/chain/tournament_object.cpp b/libraries/chain/tournament_object.cpp index c1b53f79..ad64b34f 100644 --- a/libraries/chain/tournament_object.cpp +++ b/libraries/chain/tournament_object.cpp @@ -722,37 +722,37 @@ namespace graphene { namespace chain { namespace fc { // Manually reflect tournament_object to variant to properly reflect "state" - void to_variant(const graphene::chain::tournament_object& tournament_obj, fc::variant& v) + void to_variant(const graphene::chain::tournament_object& tournament_obj, fc::variant& v, uint32_t max_depth) { fc_elog(fc::logger::get("tournament"), "In tournament_obj to_variant"); elog("In tournament_obj to_variant"); fc::mutable_variant_object o; - o("id", tournament_obj.id) - ("creator", tournament_obj.creator) - ("options", tournament_obj.options) - ("start_time", tournament_obj.start_time) - ("end_time", tournament_obj.end_time) - ("prize_pool", tournament_obj.prize_pool) - ("registered_players", tournament_obj.registered_players) - ("tournament_details_id", tournament_obj.tournament_details_id) - ("state", tournament_obj.get_state()); + o("id", fc::variant(tournament_obj.id, max_depth)) + ("creator", fc::variant(tournament_obj.creator, max_depth)) + ("options", fc::variant(tournament_obj.options, max_depth)) + ("start_time", fc::variant(tournament_obj.start_time, max_depth)) + ("end_time", fc::variant(tournament_obj.end_time, max_depth)) + ("prize_pool", fc::variant(tournament_obj.prize_pool, max_depth)) + ("registered_players", fc::variant(tournament_obj.registered_players, max_depth)) + ("tournament_details_id", fc::variant(tournament_obj.tournament_details_id, max_depth)) + ("state", fc::variant(tournament_obj.get_state(), max_depth)); v = o; } // Manually reflect tournament_object to variant to properly reflect "state" - void from_variant(const fc::variant& v, graphene::chain::tournament_object& tournament_obj) + void from_variant(const fc::variant& v, graphene::chain::tournament_object& tournament_obj, uint32_t max_depth) { fc_elog(fc::logger::get("tournament"), "In tournament_obj from_variant"); - tournament_obj.id = v["id"].as(); - tournament_obj.creator = v["creator"].as(); - tournament_obj.options = v["options"].as(); - tournament_obj.start_time = v["start_time"].as >(); - tournament_obj.end_time = v["end_time"].as >(); - tournament_obj.prize_pool = v["prize_pool"].as(); - tournament_obj.registered_players = v["registered_players"].as(); - tournament_obj.tournament_details_id = v["tournament_details_id"].as(); - graphene::chain::tournament_state state = v["state"].as(); + tournament_obj.id = v["id"].as( max_depth ); + tournament_obj.creator = v["creator"].as( max_depth ); + tournament_obj.options = v["options"].as( max_depth ); + tournament_obj.start_time = v["start_time"].as >( max_depth ); + tournament_obj.end_time = v["end_time"].as >( max_depth ); + tournament_obj.prize_pool = v["prize_pool"].as( max_depth ); + tournament_obj.registered_players = v["registered_players"].as( max_depth ); + tournament_obj.tournament_details_id = v["tournament_details_id"].as( max_depth ); + graphene::chain::tournament_state state = v["state"].as( max_depth ); const_cast(tournament_obj.my->state_machine.current_state())[0] = (int)state; } } //end namespace fc diff --git a/libraries/db/include/graphene/db/index.hpp b/libraries/db/include/graphene/db/index.hpp index 15c0f94c..1bc593f4 100644 --- a/libraries/db/include/graphene/db/index.hpp +++ b/libraries/db/include/graphene/db/index.hpp @@ -23,11 +23,13 @@ */ #pragma once #include + #include #include #include #include #include +#include namespace graphene { namespace db { class object_database; @@ -130,7 +132,7 @@ namespace graphene { namespace db { virtual fc::uint128 hash()const = 0; virtual void add_observer( const shared_ptr& ) = 0; - virtual void object_from_variant( const fc::variant& var, object& obj )const = 0; + virtual void object_from_variant( const fc::variant& var, object& obj, uint32_t max_depth )const = 0; virtual void object_default( object& obj )const = 0; }; @@ -190,7 +192,112 @@ namespace graphene { namespace db { object_database& _db; }; + /** @class direct_index + * @brief A secondary index that tracks objects in vectors indexed by object + * id. It is meant for fully (or almost fully) populated indexes only (will + * fail when loading an object_database with large gaps). + * + * WARNING! If any of the methods called on insertion, removal or + * modification throws, subsequent behaviour is undefined! Such exceptions + * indicate that this index type is not appropriate for the use-case. + */ + template + class direct_index : public secondary_index + { + static_assert( chunkbits < 64, "Do you really want arrays with more than 2^63 elements???" ); + // private + static const size_t MAX_HOLE = 100; + static const size_t _mask = ((1 << chunkbits) - 1); + size_t next = 0; + vector< vector< const Object* > > content; + std::stack< object_id_type > ids_being_modified; + + public: + direct_index() { + FC_ASSERT( (1ULL << chunkbits) > MAX_HOLE, "Small chunkbits is inefficient." ); + } + + virtual ~direct_index(){} + + virtual void object_inserted( const object& obj ) + { + uint64_t instance = obj.id.instance(); + if( instance == next ) + { + if( !(next & _mask) ) + { + content.resize((next >> chunkbits) + 1); + content[next >> chunkbits].resize( 1 << chunkbits, nullptr ); + } + next++; + } + else if( instance < next ) + FC_ASSERT( !content[instance >> chunkbits][instance & _mask], "Overwriting insert at {id}!", ("id",obj.id) ); + else // instance > next, allow small "holes" + { + FC_ASSERT( instance <= next + MAX_HOLE, "Out-of-order insert: {id} > {next}!", ("id",obj.id)("next",next) ); + if( !(next & _mask) || (next & (~_mask)) != (instance & (~_mask)) ) + { + content.resize((instance >> chunkbits) + 1); + content[instance >> chunkbits].resize( 1 << chunkbits, nullptr ); + } + while( next <= instance ) + { + content[next >> chunkbits][next & _mask] = nullptr; + next++; + } + } + FC_ASSERT( nullptr != dynamic_cast(&obj), "Wrong object type!" ); + content[instance >> chunkbits][instance & _mask] = static_cast( &obj ); + } + + virtual void object_removed( const object& obj ) + { + FC_ASSERT( nullptr != dynamic_cast(&obj), "Wrong object type!" ); + uint64_t instance = obj.id.instance(); + FC_ASSERT( instance < next, "Removing out-of-range object: {id} > {next}!", ("id",obj.id)("next",next) ); + FC_ASSERT( content[instance >> chunkbits][instance & _mask], "Removing non-existent object {id}!", ("id",obj.id) ); + content[instance >> chunkbits][instance & _mask] = nullptr; + } + + virtual void about_to_modify( const object& before ) + { + ids_being_modified.emplace( before.id ); + } + + virtual void object_modified( const object& after ) + { + FC_ASSERT( ids_being_modified.top() == after.id, "Modification of ID is not supported!"); + ids_being_modified.pop(); + } + + template< typename object_id > + const Object* find( const object_id& id )const + { + static_assert( object_id::space_id == Object::space_id, "Space ID mismatch!" ); + static_assert( object_id::type_id == Object::type_id, "Type_ID mismatch!" ); + if( id.instance >= next ) return nullptr; + return content[id.instance.value >> chunkbits][id.instance.value & _mask]; + }; + + template< typename object_id > + const Object& get( const object_id& id )const + { + const Object* ptr = find( id ); + FC_ASSERT( ptr != nullptr, "Object not found!" ); + return *ptr; + }; + + const Object* find( const object_id_type& id )const + { + FC_ASSERT( id.space() == Object::space_id, "Space ID mismatch!" ); + FC_ASSERT( id.type() == Object::type_id, "Type_ID mismatch!" ); + if( id.instance() >= next ) return nullptr; + return content[id.instance() >> chunkbits][id.instance() & ((1 << chunkbits) - 1)]; + }; + }; + /** * @class primary_index * @brief Wraps a derived index to intercept calls to create, modify, and remove so that @@ -198,14 +305,18 @@ namespace graphene { namespace db { * * @see http://en.wikipedia.org/wiki/Curiously_recurring_template_pattern */ - template + template class primary_index : public DerivedIndex, public base_primary_index { public: typedef typename DerivedIndex::object_type object_type; primary_index( object_database& db ) - :base_primary_index(db),_next_id(object_type::space_id,object_type::type_id,0) {} + :base_primary_index(db),_next_id(object_type::space_id,object_type::type_id,0) + { + if( DirectBits > 0 ) + _direct_by_id = add_secondary_index< direct_index< object_type, DirectBits > >(); + } virtual uint8_t object_space_id()const override { return object_type::space_id; } @@ -216,7 +327,15 @@ namespace graphene { namespace db { virtual object_id_type get_next_id()const override { return _next_id; } virtual void use_next_id()override { ++_next_id.number; } virtual void set_next_id( object_id_type id )override { _next_id = id; } - + + /** @return the object with id or nullptr if not found */ + virtual const object* find( object_id_type id )const override + { + if( DirectBits > 0 ) + return _direct_by_id->find( id ); + return DerivedIndex::find( id ); + } + fc::sha256 get_object_version()const { std::string desc = "1.0";//get_type_description(); @@ -299,12 +418,12 @@ namespace graphene { namespace db { _observers.emplace_back( o ); } - virtual void object_from_variant( const fc::variant& var, object& obj )const override + virtual void object_from_variant( const fc::variant& var, object& obj, uint32_t max_depth )const override { object_id_type id = obj.id; object_type* result = dynamic_cast( &obj ); FC_ASSERT( result != nullptr ); - fc::from_variant( var, *result ); + fc::from_variant( var, *result, max_depth ); obj.id = id; } @@ -318,7 +437,8 @@ namespace graphene { namespace db { } private: - object_id_type _next_id; + object_id_type _next_id; + const direct_index< object_type, DirectBits >* _direct_by_id = nullptr; }; } } // graphene::db diff --git a/libraries/db/include/graphene/db/object.hpp b/libraries/db/include/graphene/db/object.hpp index d8d16c33..c410e273 100644 --- a/libraries/db/include/graphene/db/object.hpp +++ b/libraries/db/include/graphene/db/object.hpp @@ -27,6 +27,8 @@ #include #include +#define MAX_NESTING (200) + namespace graphene { namespace db { /** @@ -98,7 +100,7 @@ namespace graphene { namespace db { { static_cast(*this) = std::move( static_cast(obj) ); } - virtual variant to_variant()const { return variant( static_cast(*this) ); } + virtual variant to_variant()const { return variant( static_cast(*this), MAX_NESTING ); } virtual vector pack()const { return fc::raw::pack( static_cast(*this) ); } virtual fc::uint128 hash()const { auto tmp = this->pack(); diff --git a/libraries/db/include/graphene/db/object_id.hpp b/libraries/db/include/graphene/db/object_id.hpp index 598ff3de..255ef048 100644 --- a/libraries/db/include/graphene/db/object_id.hpp +++ b/libraries/db/include/graphene/db/object_id.hpp @@ -169,12 +169,12 @@ struct reflector > }; - inline void to_variant( const graphene::db::object_id_type& var, fc::variant& vo ) + inline void to_variant( const graphene::db::object_id_type& var, fc::variant& vo, uint32_t max_depth = 1 ) { vo = std::string( var ); } - inline void from_variant( const fc::variant& var, graphene::db::object_id_type& vo ) + inline void from_variant( const fc::variant& var, graphene::db::object_id_type& vo, uint32_t max_depth = 1 ) { try { vo.number = 0; const auto& s = var.get_string(); @@ -191,12 +191,12 @@ struct reflector > vo.number |= (space_id << 56) | (type_id << 48); } FC_CAPTURE_AND_RETHROW( (var) ) } template - void to_variant( const graphene::db::object_id& var, fc::variant& vo ) + void to_variant( const graphene::db::object_id& var, fc::variant& vo, uint32_t max_depth = 1 ) { vo = fc::to_string(SpaceID) + "." + fc::to_string(TypeID) + "." + fc::to_string(var.instance.value); } template - void from_variant( const fc::variant& var, graphene::db::object_id& vo ) + void from_variant( const fc::variant& var, graphene::db::object_id& vo, uint32_t max_depth = 1 ) { try { const auto& s = var.get_string(); auto first_dot = s.find('.'); diff --git a/libraries/db/object_database.cpp b/libraries/db/object_database.cpp index 29d83ae7..fdde0fed 100644 --- a/libraries/db/object_database.cpp +++ b/libraries/db/object_database.cpp @@ -71,14 +71,20 @@ index& object_database::get_mutable_index(uint8_t space_id, uint8_t type_id) void object_database::flush() { // ilog("Save object_database in ${d}", ("d", _data_dir)); + fc::create_directories( _data_dir / "object_database.tmp" / "lock" ); for( uint32_t space = 0; space < _index.size(); ++space ) { - fc::create_directories( _data_dir / "object_database" / fc::to_string(space) ); + fc::create_directories( _data_dir / "object_database.tmp" / fc::to_string(space) ); const auto types = _index[space].size(); for( uint32_t type = 0; type < types; ++type ) if( _index[space][type] ) - _index[space][type]->save( _data_dir / "object_database" / fc::to_string(space)/fc::to_string(type) ); + _index[space][type]->save( _data_dir / "object_database.tmp" / fc::to_string(space)/fc::to_string(type) ); } + fc::remove_all( _data_dir / "object_database.tmp" / "lock" ); + if( fc::exists( _data_dir / "object_database" ) ) + fc::rename( _data_dir / "object_database", _data_dir / "object_database.old" ); + fc::rename( _data_dir / "object_database.tmp", _data_dir / "object_database" ); + fc::remove_all( _data_dir / "object_database.old" ); } void object_database::wipe(const fc::path& data_dir) @@ -91,8 +97,13 @@ void object_database::wipe(const fc::path& data_dir) void object_database::open(const fc::path& data_dir) { try { - ilog("Opening object database from ${d} ...", ("d", data_dir)); _data_dir = data_dir; + if( fc::exists( _data_dir / "object_database" / "lock" ) ) + { + wlog("Ignoring locked object_database"); + return; + } + ilog("Opening object database from ${d} ...", ("d", data_dir)); for( uint32_t space = 0; space < _index.size(); ++space ) for( uint32_t type = 0; type < _index[space].size(); ++type ) if( _index[space][type] ) diff --git a/libraries/db/undo_database.cpp b/libraries/db/undo_database.cpp index b37b2c7d..c5f2ef65 100644 --- a/libraries/db/undo_database.cpp +++ b/libraries/db/undo_database.cpp @@ -118,8 +118,6 @@ void undo_database::undo() _db.insert( std::move(*item.second) ); _stack.pop_back(); - if( _stack.empty() ) - _stack.emplace_back(); enable(); --_active_sessions; } FC_CAPTURE_AND_RETHROW() } @@ -127,6 +125,12 @@ void undo_database::undo() void undo_database::merge() { FC_ASSERT( _active_sessions > 0 ); + if( _active_sessions == 1 && _stack.size() == 1 ) + { + _stack.pop_back(); + --_active_sessions; + return; + } FC_ASSERT( _stack.size() >=2 ); auto& state = _stack.back(); auto& prev_state = _stack[_stack.size()-2]; diff --git a/libraries/egenesis/egenesis_brief.cpp.tmpl b/libraries/egenesis/egenesis_brief.cpp.tmpl index 8ee2ba3b..bd590eb3 100644 --- a/libraries/egenesis/egenesis_brief.cpp.tmpl +++ b/libraries/egenesis/egenesis_brief.cpp.tmpl @@ -26,7 +26,7 @@ using namespace graphene::chain; chain_id_type get_egenesis_chain_id() { - return chain_id_type( "${chain_id}$" ); + return chain_id_type( "${chain_id}" ); } void compute_egenesis_json( std::string& result ) diff --git a/libraries/egenesis/egenesis_full.cpp.tmpl b/libraries/egenesis/egenesis_full.cpp.tmpl index 7054e20f..83285f11 100644 --- a/libraries/egenesis/egenesis_full.cpp.tmpl +++ b/libraries/egenesis/egenesis_full.cpp.tmpl @@ -24,26 +24,25 @@ namespace graphene { namespace egenesis { using namespace graphene::chain; -static const char genesis_json_array[${genesis_json_array_height}$][${genesis_json_array_width}$+1] = +static const char genesis_json_array[${genesis_json_array_height}][${genesis_json_array_width}+1] = { -${genesis_json_array}$ +${genesis_json_array} }; chain_id_type get_egenesis_chain_id() { - return chain_id_type( "${chain_id}$" ); + return chain_id_type( "${chain_id}" ); } void compute_egenesis_json( std::string& result ) { - result.reserve( ${genesis_json_length}$ ); + result.reserve( ${genesis_json_length} ); result.resize(0); - for( size_t i=0; i<${genesis_json_array_height}$-1; i++ ) + for( size_t i=0; i<${genesis_json_array_height}-1; i++ ) { - result.append( genesis_json_array[i], ${genesis_json_array_width}$ ); + result.append( genesis_json_array[i], ${genesis_json_array_width} ); } - result.append( std::string( genesis_json_array[ ${genesis_json_array_height}$-1 ] ) ); - return; + result.append( std::string( genesis_json_array[ ${genesis_json_array_height}-1 ] ) ); } fc::sha256 get_egenesis_json_hash() diff --git a/libraries/egenesis/embed_genesis.cpp b/libraries/egenesis/embed_genesis.cpp index 6fac5dbb..26283080 100644 --- a/libraries/egenesis/embed_genesis.cpp +++ b/libraries/egenesis/embed_genesis.cpp @@ -168,7 +168,7 @@ struct egenesis_info // If genesis not exist, generate from genesis_json try { - genesis = fc::json::from_string( *genesis_json ).as< genesis_state_type >(); + genesis = fc::json::from_string( *genesis_json ).as< genesis_state_type >( 20 ); } catch (const fc::exception& e) { @@ -223,7 +223,6 @@ void load_genesis( std::cerr << "embed_genesis: Genesis ID from argument is " << chain_id_str << "\n"; info.chain_id = chain_id_str; } - return; } int main( int argc, char** argv ) diff --git a/libraries/fc b/libraries/fc index 443f858d..f13d0632 160000 --- a/libraries/fc +++ b/libraries/fc @@ -1 +1 @@ -Subproject commit 443f858d9b4733bb6d894da9315ce00ac3246065 +Subproject commit f13d0632b08b9983a275304317a033914938e339 diff --git a/libraries/net/include/graphene/net/config.hpp b/libraries/net/include/graphene/net/config.hpp index 1d400bcf..9edca51c 100644 --- a/libraries/net/include/graphene/net/config.hpp +++ b/libraries/net/include/graphene/net/config.hpp @@ -106,3 +106,7 @@ #define GRAPHENE_NET_MIN_BLOCK_IDS_TO_PREFETCH 10000 #define GRAPHENE_NET_MAX_TRX_PER_SECOND 1000 + +#define GRAPHENE_NET_MAX_NESTED_OBJECTS (250) + +#define MAXIMUM_PEERDB_SIZE 1000 diff --git a/libraries/net/node.cpp b/libraries/net/node.cpp index d3f90425..a38199fd 100644 --- a/libraries/net/node.cpp +++ b/libraries/net/node.cpp @@ -1858,10 +1858,10 @@ namespace graphene { namespace net { namespace detail { #endif user_data["bitness"] = sizeof(void*) * 8; - user_data["node_id"] = _node_id; + user_data["node_id"] = fc::variant( _node_id, 1 ); item_hash_t head_block_id = _delegate->get_head_block_id(); - user_data["last_known_block_hash"] = head_block_id; + user_data["last_known_block_hash"] = fc::variant( head_block_id, 1 ); user_data["last_known_block_number"] = _delegate->get_block_number(head_block_id); user_data["last_known_block_time"] = _delegate->get_block_time(head_block_id); @@ -1877,19 +1877,19 @@ namespace graphene { namespace net { namespace detail { if (user_data.contains("graphene_git_revision_sha")) originating_peer->graphene_git_revision_sha = user_data["graphene_git_revision_sha"].as_string(); if (user_data.contains("graphene_git_revision_unix_timestamp")) - originating_peer->graphene_git_revision_unix_timestamp = fc::time_point_sec(user_data["graphene_git_revision_unix_timestamp"].as()); + originating_peer->graphene_git_revision_unix_timestamp = fc::time_point_sec(user_data["graphene_git_revision_unix_timestamp"].as(1)); if (user_data.contains("fc_git_revision_sha")) originating_peer->fc_git_revision_sha = user_data["fc_git_revision_sha"].as_string(); if (user_data.contains("fc_git_revision_unix_timestamp")) - originating_peer->fc_git_revision_unix_timestamp = fc::time_point_sec(user_data["fc_git_revision_unix_timestamp"].as()); + originating_peer->fc_git_revision_unix_timestamp = fc::time_point_sec(user_data["fc_git_revision_unix_timestamp"].as(1)); if (user_data.contains("platform")) originating_peer->platform = user_data["platform"].as_string(); if (user_data.contains("bitness")) - originating_peer->bitness = user_data["bitness"].as(); + originating_peer->bitness = user_data["bitness"].as(1); if (user_data.contains("node_id")) - originating_peer->node_id = user_data["node_id"].as(); + originating_peer->node_id = user_data["node_id"].as(1); if (user_data.contains("last_known_fork_block_number")) - originating_peer->last_known_fork_block_number = user_data["last_known_fork_block_number"].as(); + originating_peer->last_known_fork_block_number = user_data["last_known_fork_block_number"].as(1); } void node_impl::on_hello_message( peer_connection* originating_peer, const hello_message& hello_message_received ) @@ -1899,7 +1899,7 @@ namespace graphene { namespace net { namespace detail { node_id_t peer_node_id = hello_message_received.node_public_key; try { - peer_node_id = hello_message_received.user_data["node_id"].as(); + peer_node_id = hello_message_received.user_data["node_id"].as(1); } catch (const fc::exception&) { @@ -2940,7 +2940,7 @@ namespace graphene { namespace net { namespace detail { ( "msg", closing_connection_message_received.reason_for_closing ) ( "error", closing_connection_message_received.error ) ); std::ostringstream message; - message << "Peer " << fc::variant( originating_peer->get_remote_endpoint() ).as_string() << + message << "Peer " << fc::variant( originating_peer->get_remote_endpoint(), GRAPHENE_NET_MAX_NESTED_OBJECTS ).as_string() << " disconnected us: " << closing_connection_message_received.reason_for_closing; fc::exception detailed_error(FC_LOG_MESSAGE(warn, "Peer ${peer} is disconnecting us because of an error: ${msg}, exception: ${error}", ( "peer", originating_peer->get_remote_endpoint() ) @@ -3846,7 +3846,7 @@ namespace graphene { namespace net { namespace detail { user_data["bitness"] = *peer->bitness; user_data["user_agent"] = peer->user_agent; - user_data["last_known_block_hash"] = peer->last_block_delegate_has_seen; + user_data["last_known_block_hash"] = fc::variant( peer->last_block_delegate_has_seen, 1 ); user_data["last_known_block_number"] = _delegate->get_block_number(peer->last_block_delegate_has_seen); user_data["last_known_block_time"] = peer->last_block_time_delegate_has_seen; @@ -4457,7 +4457,7 @@ namespace graphene { namespace net { namespace detail { { try { - _node_configuration = fc::json::from_file( configuration_file_name ).as(); + _node_configuration = fc::json::from_file( configuration_file_name ).as(GRAPHENE_NET_MAX_NESTED_OBJECTS); ilog( "Loaded configuration from file ${filename}", ("filename", configuration_file_name ) ); if( _node_configuration.private_key == fc::ecc::private_key() ) @@ -4821,20 +4821,19 @@ namespace graphene { namespace net { namespace detail { peer_to_disconnect->send_message( closing_message ); } - // notify the user. This will be useful in testing, but we might want to remove it later; - // it makes good sense to notify the user if other nodes think she is behaving badly, but + // notify the user. This will be useful in testing, but we might want to remove it later. + // It makes good sense to notify the user if other nodes think she is behaving badly, but // if we're just detecting and dissconnecting other badly-behaving nodes, they don't really care. if (caused_by_error) { std::ostringstream error_message; - error_message << "I am disconnecting peer " << fc::variant( peer_to_disconnect->get_remote_endpoint() ).as_string() << + error_message << "I am disconnecting peer " << fc::variant( peer_to_disconnect->get_remote_endpoint(), GRAPHENE_NET_MAX_NESTED_OBJECTS ).as_string() << " for reason: " << reason_for_disconnect; _delegate->error_encountered(error_message.str(), fc::oexception()); dlog(error_message.str()); } else dlog("Disconnecting from ${peer} for ${reason}", ("peer",peer_to_disconnect->get_remote_endpoint()) ("reason",reason_for_disconnect)); - // peer_to_disconnect->close_connection(); } void node_impl::listen_on_endpoint( const fc::ip::endpoint& ep, bool wait_if_not_available ) @@ -4893,7 +4892,7 @@ namespace graphene { namespace net { namespace detail { peer_details["version"] = ""; peer_details["subver"] = peer->user_agent; peer_details["inbound"] = peer->direction == peer_connection_direction::inbound; - peer_details["firewall_status"] = peer->is_firewalled; + peer_details["firewall_status"] = fc::variant( peer->is_firewalled, 1 ); peer_details["startingheight"] = ""; peer_details["banscore"] = ""; peer_details["syncnode"] = ""; @@ -4927,7 +4926,7 @@ namespace graphene { namespace net { namespace detail { // provide these for debugging // warning: these are just approximations, if the peer is "downstream" of us, they may // have received blocks from other peers that we are unaware of - peer_details["current_head_block"] = peer->last_block_delegate_has_seen; + peer_details["current_head_block"] = fc::variant( peer->last_block_delegate_has_seen, 1 ); peer_details["current_head_block_number"] = _delegate->get_block_number(peer->last_block_delegate_has_seen); peer_details["current_head_block_time"] = peer->last_block_time_delegate_has_seen; @@ -5003,17 +5002,17 @@ namespace graphene { namespace net { namespace detail { { VERIFY_CORRECT_THREAD(); if (params.contains("peer_connection_retry_timeout")) - _peer_connection_retry_timeout = params["peer_connection_retry_timeout"].as(); + _peer_connection_retry_timeout = params["peer_connection_retry_timeout"].as(1); if (params.contains("desired_number_of_connections")) - _desired_number_of_connections = params["desired_number_of_connections"].as(); + _desired_number_of_connections = params["desired_number_of_connections"].as(1); if (params.contains("maximum_number_of_connections")) - _maximum_number_of_connections = params["maximum_number_of_connections"].as(); + _maximum_number_of_connections = params["maximum_number_of_connections"].as(1); if (params.contains("maximum_number_of_blocks_to_handle_at_one_time")) - _maximum_number_of_blocks_to_handle_at_one_time = params["maximum_number_of_blocks_to_handle_at_one_time"].as(); + _maximum_number_of_blocks_to_handle_at_one_time = params["maximum_number_of_blocks_to_handle_at_one_time"].as(1); if (params.contains("maximum_number_of_sync_blocks_to_prefetch")) - _maximum_number_of_sync_blocks_to_prefetch = params["maximum_number_of_sync_blocks_to_prefetch"].as(); + _maximum_number_of_sync_blocks_to_prefetch = params["maximum_number_of_sync_blocks_to_prefetch"].as(1); if (params.contains("maximum_blocks_per_peer_during_syncing")) - _maximum_blocks_per_peer_during_syncing = params["maximum_blocks_per_peer_during_syncing"].as(); + _maximum_blocks_per_peer_during_syncing = params["maximum_blocks_per_peer_during_syncing"].as(1); _desired_number_of_connections = std::min(_desired_number_of_connections, _maximum_number_of_connections); @@ -5098,9 +5097,9 @@ namespace graphene { namespace net { namespace detail { VERIFY_CORRECT_THREAD(); fc::mutable_variant_object info; info["listening_on"] = _actual_listening_endpoint; - info["node_public_key"] = _node_public_key; - info["node_id"] = _node_id; - info["firewalled"] = _is_firewalled; + info["node_public_key"] = fc::variant( _node_public_key, 1 ); + info["node_id"] = fc::variant( _node_id, 1 ); + info["firewalled"] = fc::variant( _is_firewalled, 1 ); return info; } fc::variant_object node_impl::network_get_usage_stats() const @@ -5128,9 +5127,9 @@ namespace graphene { namespace net { namespace detail { std::plus()); fc::mutable_variant_object result; - result["usage_by_second"] = network_usage_by_second; - result["usage_by_minute"] = network_usage_by_minute; - result["usage_by_hour"] = network_usage_by_hour; + result["usage_by_second"] = fc::variant( network_usage_by_second, 2 ); + result["usage_by_minute"] = fc::variant( network_usage_by_minute, 2 ); + result["usage_by_hour"] = fc::variant( network_usage_by_hour, 2 ); return result; } diff --git a/libraries/net/peer_database.cpp b/libraries/net/peer_database.cpp index c24568fc..2b20364e 100644 --- a/libraries/net/peer_database.cpp +++ b/libraries/net/peer_database.cpp @@ -34,8 +34,7 @@ #include #include - - +#include namespace graphene { namespace net { namespace detail @@ -81,7 +80,7 @@ namespace graphene { namespace net { public: typedef peer_database_impl::potential_peer_set::index::type::iterator last_seen_time_index_iterator; last_seen_time_index_iterator _iterator; - peer_database_iterator_impl(const last_seen_time_index_iterator& iterator) : + explicit peer_database_iterator_impl(const last_seen_time_index_iterator& iterator) : _iterator(iterator) {} }; @@ -95,9 +94,8 @@ namespace graphene { namespace net { { try { - std::vector peer_records = fc::json::from_file(_peer_database_filename).as >(); + std::vector peer_records = fc::json::from_file(_peer_database_filename).as >( GRAPHENE_NET_MAX_NESTED_OBJECTS ); std::copy(peer_records.begin(), peer_records.end(), std::inserter(_potential_peer_set, _potential_peer_set.end())); -#define MAXIMUM_PEERDB_SIZE 1000 if (_potential_peer_set.size() > MAXIMUM_PEERDB_SIZE) { // prune database to a reasonable size @@ -125,7 +123,7 @@ namespace graphene { namespace net { fc::path peer_database_filename_dir = _peer_database_filename.parent_path(); if (!fc::exists(peer_database_filename_dir)) fc::create_directories(peer_database_filename_dir); - fc::json::save_to_file(peer_records, _peer_database_filename); + fc::json::save_to_file( peer_records, _peer_database_filename, GRAPHENE_NET_MAX_NESTED_OBJECTS ); } catch (const fc::exception& e) { diff --git a/libraries/plugins/debug_witness/debug_api.cpp b/libraries/plugins/debug_witness/debug_api.cpp index 6236482b..823755f5 100644 --- a/libraries/plugins/debug_witness/debug_api.cpp +++ b/libraries/plugins/debug_witness/debug_api.cpp @@ -22,12 +22,11 @@ namespace detail { class debug_api_impl { public: - debug_api_impl( graphene::app::application& _app ); + explicit debug_api_impl( graphene::app::application& _app ); void debug_push_blocks( const std::string& src_filename, uint32_t count ); void debug_generate_blocks( const std::string& debug_key, uint32_t count ); void debug_update_object( const fc::variant_object& update ); - //void debug_save_db( std::string db_path ); void debug_stream_json_objects( const std::string& filename ); void debug_stream_json_objects_flush(); std::shared_ptr< graphene::debug_witness_plugin::debug_witness_plugin > get_plugin(); @@ -71,7 +70,6 @@ void debug_api_impl::debug_push_blocks( const std::string& src_filename, uint32_ } } ilog( "Completed loading block_database successfully" ); - return; } } @@ -93,7 +91,7 @@ void debug_api_impl::debug_generate_blocks( const std::string& debug_key, uint32 if( scheduled_key != debug_public_key ) { ilog( "Modified key for witness ${w}", ("w", scheduled_witness) ); - fc::mutable_variant_object update; + fc::limited_mutable_variant_object update( GRAPHENE_MAX_NESTED_OBJECTS ); update("_action", "update")("id", scheduled_witness)("signing_key", debug_public_key); db->debug_update( update ); } diff --git a/libraries/plugins/debug_witness/debug_witness.cpp b/libraries/plugins/debug_witness/debug_witness.cpp index 17f852c5..aea03e32 100644 --- a/libraries/plugins/debug_witness/debug_witness.cpp +++ b/libraries/plugins/debug_witness/debug_witness.cpp @@ -68,7 +68,7 @@ void debug_witness_plugin::plugin_initialize(const boost::program_options::varia const std::vector key_id_to_wif_pair_strings = options["private-key"].as>(); for (const std::string& key_id_to_wif_pair_string : key_id_to_wif_pair_strings) { - auto key_id_to_wif_pair = graphene::app::dejsonify >(key_id_to_wif_pair_string); + auto key_id_to_wif_pair = graphene::app::dejsonify >(key_id_to_wif_pair_string, GRAPHENE_MAX_NESTED_OBJECTS); idump((key_id_to_wif_pair)); fc::optional private_key = graphene::utilities::wif_to_key(key_id_to_wif_pair.second); if (!private_key) @@ -77,7 +77,7 @@ void debug_witness_plugin::plugin_initialize(const boost::program_options::varia // just here to ease the transition, can be removed soon try { - private_key = fc::variant(key_id_to_wif_pair.second).as(); + private_key = fc::variant( key_id_to_wif_pair.second, GRAPHENE_MAX_NESTED_OBJECTS ).as( GRAPHENE_MAX_NESTED_OBJECTS ); } catch (const fc::exception&) { diff --git a/libraries/plugins/delayed_node/delayed_node_plugin.cpp b/libraries/plugins/delayed_node/delayed_node_plugin.cpp index 203181bd..3eadda34 100644 --- a/libraries/plugins/delayed_node/delayed_node_plugin.cpp +++ b/libraries/plugins/delayed_node/delayed_node_plugin.cpp @@ -65,7 +65,7 @@ void delayed_node_plugin::plugin_set_program_options(bpo::options_description& c void delayed_node_plugin::connect() { - my->client_connection = std::make_shared(my->client.connect(my->remote_endpoint)); + my->client_connection = std::make_shared(my->client.connect(my->remote_endpoint), GRAPHENE_MAX_NESTED_OBJECTS); my->database_api = my->client_connection->get_remote_api(0); my->client_connection_closed = my->client_connection->closed.connect([this] { connection_failed(); @@ -143,7 +143,7 @@ void delayed_node_plugin::plugin_startup() connect(); my->database_api->set_block_applied_callback([this]( const fc::variant& block_id ) { - fc::from_variant( block_id, my->last_received_remote_head ); + fc::from_variant( block_id, my->last_received_remote_head, GRAPHENE_MAX_NESTED_OBJECTS ); } ); return; } diff --git a/libraries/plugins/grouped_orders/grouped_orders_plugin.cpp b/libraries/plugins/grouped_orders/grouped_orders_plugin.cpp new file mode 100644 index 00000000..ef1ae04c --- /dev/null +++ b/libraries/plugins/grouped_orders/grouped_orders_plugin.cpp @@ -0,0 +1,303 @@ +/* + * Copyright (c) 2018 Abit More, and contributors. + * + * The MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include + +#include + +namespace graphene { namespace grouped_orders { + +namespace detail +{ + +class grouped_orders_plugin_impl +{ + public: + grouped_orders_plugin_impl(grouped_orders_plugin& _plugin) + :_self( _plugin ) {} + virtual ~grouped_orders_plugin_impl(); + + graphene::chain::database& database() + { + return _self.database(); + } + + grouped_orders_plugin& _self; + flat_set _tracked_groups; +}; + +/** + * @brief This secondary index is used to track changes on limit order objects. + */ +class limit_order_group_index : public secondary_index +{ + public: + limit_order_group_index( const flat_set& groups ) : _tracked_groups( groups ) {}; + + virtual void object_inserted( const object& obj ) override; + virtual void object_removed( const object& obj ) override; + virtual void about_to_modify( const object& before ) override; + virtual void object_modified( const object& after ) override; + + const flat_set& get_tracked_groups() const + { return _tracked_groups; } + + const map< limit_order_group_key, limit_order_group_data >& get_order_groups() const + { return _og_data; } + + private: + void remove_order( const limit_order_object& obj, bool remove_empty = true ); + + /** tracked groups */ + flat_set _tracked_groups; + + /** maps the group key to group data */ + map< limit_order_group_key, limit_order_group_data > _og_data; +}; + +void limit_order_group_index::object_inserted( const object& objct ) +{ try { + const limit_order_object& o = static_cast( objct ); + + auto& idx = _og_data; + + for( uint16_t group : get_tracked_groups() ) + { + auto create_ogo = [&]() { + idx[ limit_order_group_key( group, o.sell_price ) ] = limit_order_group_data( o.sell_price, o.for_sale ); + }; + // if idx is empty, insert this order + // Note: not capped + if( idx.empty() ) + { + create_ogo(); + continue; + } + + // cap the price + price capped_price = o.sell_price; + price max = o.sell_price.max(); + price min = o.sell_price.min(); + bool capped_max = false; + bool capped_min = false; + if( o.sell_price > max ) + { + capped_price = max; + capped_max = true; + } + else if( o.sell_price < min ) + { + capped_price = min; + capped_min = true; + } + // if idx is not empty, find the group that is next to this order + auto itr = idx.lower_bound( limit_order_group_key( group, capped_price ) ); + bool check_previous = false; + if( itr == idx.end() || itr->first.group != group + || itr->first.min_price.base.asset_id != o.sell_price.base.asset_id + || itr->first.min_price.quote.asset_id != o.sell_price.quote.asset_id ) + // not same market or group type + check_previous = true; + else // same market and group type + { + bool update_max = false; + if( capped_price > itr->second.max_price ) // implies itr->min_price <= itr->max_price < max + { + update_max = true; + price max_price = itr->first.min_price * ratio_type( GRAPHENE_100_PERCENT + group, GRAPHENE_100_PERCENT ); + // max_price should have been capped here + if( capped_price > max_price ) // new order is out of range + check_previous = true; + } + if( !check_previous ) // new order is within the range + { + if( capped_min && o.sell_price < itr->first.min_price ) + { // need to update itr->min_price here, if itr is below min, and new order is even lower + // TODO improve performance + limit_order_group_data data( itr->second.max_price, o.for_sale + itr->second.total_for_sale ); + idx.erase( itr ); + idx[ limit_order_group_key( group, o.sell_price ) ] = data; + } + else + { + if( update_max || ( capped_max && o.sell_price > itr->second.max_price ) ) + itr->second.max_price = o.sell_price; // store real price here, not capped + itr->second.total_for_sale += o.for_sale; + } + } + } + + if( check_previous ) + { + if( itr == idx.begin() ) // no previous + create_ogo(); + else + { + --itr; // should be valid + if( itr->first.group != group || itr->first.min_price.base.asset_id != o.sell_price.base.asset_id + || itr->first.min_price.quote.asset_id != o.sell_price.quote.asset_id ) + // not same market or group type + create_ogo(); + else // same market and group type + { + // due to lower_bound, always true: capped_price < itr->first.min_price, so no need to check again, + // if new order is in range of itr group, always need to update itr->first.min_price, unless + // o.sell_price is higher than max + price min_price = itr->second.max_price / ratio_type( GRAPHENE_100_PERCENT + group, GRAPHENE_100_PERCENT ); + // min_price should have been capped here + if( capped_price < min_price ) // new order is out of range + create_ogo(); + else if( capped_max && o.sell_price >= itr->first.min_price ) + { // itr is above max, and price of new order is even higher + if( o.sell_price > itr->second.max_price ) + itr->second.max_price = o.sell_price; + itr->second.total_for_sale += o.for_sale; + } + else + { // new order is within the range + // TODO improve performance + limit_order_group_data data( itr->second.max_price, o.for_sale + itr->second.total_for_sale ); + idx.erase( itr ); + idx[ limit_order_group_key( group, o.sell_price ) ] = data; + } + } + } + } + } +} FC_CAPTURE_AND_RETHROW( (objct) ); } + +void limit_order_group_index::object_removed( const object& objct ) +{ try { + const limit_order_object& o = static_cast( objct ); + remove_order( o ); +} FC_CAPTURE_AND_RETHROW( (objct) ); } + +void limit_order_group_index::about_to_modify( const object& objct ) +{ try { + const limit_order_object& o = static_cast( objct ); + remove_order( o, false ); +} FC_CAPTURE_AND_RETHROW( (objct) ); } + +void limit_order_group_index::object_modified( const object& objct ) +{ try { + object_inserted( objct ); +} FC_CAPTURE_AND_RETHROW( (objct) ); } + +void limit_order_group_index::remove_order( const limit_order_object& o, bool remove_empty ) +{ + auto& idx = _og_data; + + for( uint16_t group : get_tracked_groups() ) + { + // find the group that should contain this order + auto itr = idx.lower_bound( limit_order_group_key( group, o.sell_price ) ); + if( itr == idx.end() || itr->first.group != group + || itr->first.min_price.base.asset_id != o.sell_price.base.asset_id + || itr->first.min_price.quote.asset_id != o.sell_price.quote.asset_id + || itr->second.max_price < o.sell_price ) + { + // can not find corresponding group, should not happen + wlog( "can not find the order group containing order for removing (price dismatch): ${o}", ("o",o) ); + continue; + } + else // found + { + if( itr->second.total_for_sale < o.for_sale ) + // should not happen + wlog( "can not find the order group containing order for removing (amount dismatch): ${o}", ("o",o) ); + else if( !remove_empty || itr->second.total_for_sale > o.for_sale ) + itr->second.total_for_sale -= o.for_sale; + else + // it's the only order in the group and need to be removed + idx.erase( itr ); + } + } +} + +grouped_orders_plugin_impl::~grouped_orders_plugin_impl() +{} + +} // end namespace detail + + +grouped_orders_plugin::grouped_orders_plugin() : + my( new detail::grouped_orders_plugin_impl(*this) ) +{ +} + +grouped_orders_plugin::~grouped_orders_plugin() +{ +} + +std::string grouped_orders_plugin::plugin_name()const +{ + return "grouped_orders"; +} + +void grouped_orders_plugin::plugin_set_program_options( + boost::program_options::options_description& cli, + boost::program_options::options_description& cfg + ) +{ + cli.add_options() + ("tracked-groups", boost::program_options::value()->default_value("[10,100]"), // 0.1% and 1% + "Group orders by percentage increase on price. Specify a JSON array of numbers here, each number is a group, number 1 means 0.01%. ") + ; + cfg.add(cli); +} + +void grouped_orders_plugin::plugin_initialize(const boost::program_options::variables_map& options) +{ try { + + if( options.count( "tracked-groups" ) ) + { + const std::string& groups = options["tracked-groups"].as(); + my->_tracked_groups = fc::json::from_string(groups).as>( 2 ); + my->_tracked_groups.erase( 0 ); + } + else + my->_tracked_groups = fc::json::from_string("[10,100]").as>(2); + + database().add_secondary_index< primary_index, detail::limit_order_group_index >( my->_tracked_groups ); + +} FC_CAPTURE_AND_RETHROW() } + +void grouped_orders_plugin::plugin_startup() +{ +} + +const flat_set& grouped_orders_plugin::tracked_groups() const +{ + return my->_tracked_groups; +} + +const map< limit_order_group_key, limit_order_group_data >& grouped_orders_plugin::limit_order_groups() +{ + const auto& idx = database().get_index_type< limit_order_index >(); + const auto& pidx = dynamic_cast&>(idx); + const auto& logidx = pidx.get_secondary_index< detail::limit_order_group_index >(); + return logidx.get_order_groups(); +} + +} } diff --git a/libraries/plugins/market_history/market_history_plugin.cpp b/libraries/plugins/market_history/market_history_plugin.cpp index 6ec38968..80876a48 100644 --- a/libraries/plugins/market_history/market_history_plugin.cpp +++ b/libraries/plugins/market_history/market_history_plugin.cpp @@ -265,14 +265,15 @@ void market_history_plugin::plugin_set_program_options( void market_history_plugin::plugin_initialize(const boost::program_options::variables_map& options) { try { - database().applied_block.connect( [&]( const signed_block& b){ my->update_market_histories(b); } ); + database().applied_block.connect( [this]( const signed_block& b){ my->update_market_histories(b); } ); database().add_index< primary_index< bucket_index > >(); database().add_index< primary_index< history_index > >(); if( options.count( "bucket-size" ) ) { - const std::string& buckets = options["bucket-size"].as(); - my->_tracked_buckets = fc::json::from_string(buckets).as>(); + const std::string& buckets = options["bucket-size"].as(); + my->_tracked_buckets = fc::json::from_string(buckets).as>(2); + my->_tracked_buckets.erase( 0 ); } if( options.count( "history-per-size" ) ) my->_maximum_history_per_bucket_size = options["history-per-size"].as(); diff --git a/libraries/plugins/witness/include/graphene/witness/witness.hpp b/libraries/plugins/witness/include/graphene/witness/witness.hpp index e2f60bf8..f0c3ece7 100644 --- a/libraries/plugins/witness/include/graphene/witness/witness.hpp +++ b/libraries/plugins/witness/include/graphene/witness/witness.hpp @@ -75,7 +75,7 @@ public: private: void schedule_production_loop(); block_production_condition::block_production_condition_enum block_production_loop(); - block_production_condition::block_production_condition_enum maybe_produce_block( fc::mutable_variant_object& capture ); + block_production_condition::block_production_condition_enum maybe_produce_block( fc::limited_mutable_variant_object& capture ); boost::program_options::variables_map _options; bool _production_enabled = false; diff --git a/libraries/plugins/witness/witness.cpp b/libraries/plugins/witness/witness.cpp index 7d862fcb..6ef7798b 100644 --- a/libraries/plugins/witness/witness.cpp +++ b/libraries/plugins/witness/witness.cpp @@ -59,7 +59,6 @@ void new_chain_banner( const graphene::chain::database& db ) "\n" ; } - return; } void witness_plugin::plugin_set_program_options( @@ -94,15 +93,14 @@ void witness_plugin::plugin_initialize(const boost::program_options::variables_m _options = &options; LOAD_VALUE_SET(options, "witness-id", _witnesses, chain::witness_id_type) if (options.count("witness-ids")) - boost::insert(_witnesses, fc::json::from_string(options.at("witness-ids").as()).as>()); + boost::insert(_witnesses, fc::json::from_string(options.at("witness-ids").as()).as>( 5 )); if( options.count("private-key") ) { const std::vector key_id_to_wif_pair_strings = options["private-key"].as>(); for (const std::string& key_id_to_wif_pair_string : key_id_to_wif_pair_strings) { - auto key_id_to_wif_pair = graphene::app::dejsonify >(key_id_to_wif_pair_string); - //idump((key_id_to_wif_pair)); + auto key_id_to_wif_pair = graphene::app::dejsonify >(key_id_to_wif_pair_string, 5); ilog("Public Key: ${public}", ("public", key_id_to_wif_pair.first)); fc::optional private_key = graphene::utilities::wif_to_key(key_id_to_wif_pair.second); if (!private_key) @@ -111,7 +109,7 @@ void witness_plugin::plugin_initialize(const boost::program_options::variables_m // just here to ease the transition, can be removed soon try { - private_key = fc::variant(key_id_to_wif_pair.second).as(); + private_key = fc::variant(key_id_to_wif_pair.second, 2).as(1); } catch (const fc::exception&) { @@ -147,7 +145,7 @@ void witness_plugin::plugin_startup() void witness_plugin::plugin_shutdown() { - return; + // nothing to do } void witness_plugin::schedule_production_loop() @@ -161,7 +159,6 @@ void witness_plugin::schedule_production_loop() fc::time_point next_wakeup( now + fc::microseconds( time_to_next_second ) ); - //wdump( (now.time_since_epoch().count())(next_wakeup.time_since_epoch().count()) ); _block_production_task = fc::schedule([this]{block_production_loop();}, next_wakeup, "Witness Block Production"); } @@ -169,7 +166,7 @@ void witness_plugin::schedule_production_loop() block_production_condition::block_production_condition_enum witness_plugin::block_production_loop() { block_production_condition::block_production_condition_enum result; - fc::mutable_variant_object capture; + fc::limited_mutable_variant_object capture( GRAPHENE_MAX_NESTED_OBJECTS ); try { result = maybe_produce_block(capture); @@ -197,10 +194,8 @@ block_production_condition::block_production_condition_enum witness_plugin::bloc ilog("Not producing block because production is disabled until we receive a recent block (see: --enable-stale-production)"); break; case block_production_condition::not_my_turn: - //ilog("Not producing block because it isn't my turn"); break; case block_production_condition::not_time_yet: - //dlog("Not producing block because slot has not yet arrived"); break; case block_production_condition::no_private_key: ilog("Not producing block because I don't have the private key for ${scheduled_key}", @@ -217,7 +212,7 @@ block_production_condition::block_production_condition_enum witness_plugin::bloc elog("Not producing block because the last block was generated by the same witness.\nThis node is probably disconnected from the network so block production has been disabled.\nDisable this check with --allow-consecutive option."); break; case block_production_condition::exception_producing_block: - elog( "exception prodcing block" ); + elog( "exception producing block" ); break; } @@ -225,7 +220,7 @@ block_production_condition::block_production_condition_enum witness_plugin::bloc return result; } -block_production_condition::block_production_condition_enum witness_plugin::maybe_produce_block( fc::mutable_variant_object& capture ) +block_production_condition::block_production_condition_enum witness_plugin::maybe_produce_block( fc::limited_mutable_variant_object& capture ) { chain::database& db = database(); fc::time_point now_fine = fc::time_point::now(); diff --git a/libraries/utilities/key_conversion.cpp b/libraries/utilities/key_conversion.cpp index e4130789..268b2b5e 100644 --- a/libraries/utilities/key_conversion.cpp +++ b/libraries/utilities/key_conversion.cpp @@ -58,7 +58,7 @@ fc::optional wif_to_key( const std::string& wif_key ) if (wif_bytes.size() < 5) return fc::optional(); std::vector key_bytes(wif_bytes.begin() + 1, wif_bytes.end() - 4); - fc::ecc::private_key key = fc::variant(key_bytes).as(); + fc::ecc::private_key key = fc::variant( key_bytes, 1 ).as( 1 ); fc::sha256 check = fc::sha256::hash(wif_bytes.data(), wif_bytes.size() - 4); fc::sha256 check2 = fc::sha256::hash(check); diff --git a/libraries/wallet/include/graphene/wallet/reflect_util.hpp b/libraries/wallet/include/graphene/wallet/reflect_util.hpp index b8d38473..44c0f412 100644 --- a/libraries/wallet/include/graphene/wallet/reflect_util.hpp +++ b/libraries/wallet/include/graphene/wallet/reflect_util.hpp @@ -39,7 +39,6 @@ namespace impl { std::string clean_name( const std::string& name ) { - std::string result; const static std::string prefix = "graphene::chain::"; const static std::string suffix = "_operation"; // graphene::chain::.*_operation @@ -81,24 +80,25 @@ struct from_which_visitor result_type operator()( const Member& dummy ) { Member result; - from_variant( v, result ); + from_variant( v, result, _max_depth ); return result; // converted from StaticVariant to Result automatically due to return type } const variant& v; + const uint32_t _max_depth; - from_which_visitor( const variant& _v ) : v(_v) {} + from_which_visitor( const variant& _v, uint32_t max_depth ) : v(_v), _max_depth(max_depth) {} }; } // namespace impl template< typename T > -T from_which_variant( int which, const variant& v ) +T from_which_variant( int which, const variant& v, uint32_t max_depth ) { // Parse a variant for a known which() T dummy; dummy.set_which( which ); - impl::from_which_visitor< T > vtor(v); + impl::from_which_visitor< T > vtor(v, max_depth); return dummy.visit( vtor ); } diff --git a/libraries/wallet/include/graphene/wallet/wallet.hpp b/libraries/wallet/include/graphene/wallet/wallet.hpp index 5618d26a..3059f179 100644 --- a/libraries/wallet/include/graphene/wallet/wallet.hpp +++ b/libraries/wallet/include/graphene/wallet/wallet.hpp @@ -34,8 +34,8 @@ using namespace std; namespace fc { - void to_variant(const account_multi_index_type& accts, variant& vo); - void from_variant(const variant &var, account_multi_index_type &vo); + void to_variant( const account_multi_index_type& accts, variant& vo, uint32_t max_depth ); + void from_variant( const variant &var, account_multi_index_type &vo, uint32_t max_depth ); } namespace graphene { namespace wallet { @@ -348,6 +348,12 @@ class wallet_api * @returns the list of asset objects, ordered by symbol */ vector list_assets(const string& lowerbound, uint32_t limit)const; + + /** Returns assets count registered on the blockchain. + * + * @returns assets count + */ + uint64_t get_asset_count()const; vector get_lotteries( asset_id_type stop = asset_id_type(), @@ -1925,6 +1931,7 @@ FC_API( graphene::wallet::wallet_api, (list_accounts) (list_account_balances) (list_assets) + (get_asset_count) (import_key) (import_accounts) (import_account_keys) diff --git a/libraries/wallet/wallet.cpp b/libraries/wallet/wallet.cpp index 0df82e7d..5d534536 100644 --- a/libraries/wallet/wallet.cpp +++ b/libraries/wallet/wallet.cpp @@ -98,7 +98,7 @@ namespace detail { struct operation_result_printer { public: - operation_result_printer( const wallet_api_impl& w ) + explicit operation_result_printer( const wallet_api_impl& w ) : _wallet(w) {} const wallet_api_impl& _wallet; typedef std::string result_type; @@ -151,10 +151,10 @@ optional maybe_id( const string& name_or_id ) { try { - return fc::variant(name_or_id).as(); + return fc::variant(name_or_id, 1).as(1); } catch (const fc::exception&) - { + { // not an ID } } return optional(); @@ -378,8 +378,8 @@ private: { try { - object_id_type id = changed_object_variant["id"].as(); - tournament_object current_tournament_obj = changed_object_variant.as(); + object_id_type id = changed_object_variant["id"].as( GRAPHENE_MAX_NESTED_OBJECTS ); + tournament_object current_tournament_obj = changed_object_variant.as( GRAPHENE_MAX_NESTED_OBJECTS ); auto tournament_cache_iter = tournament_cache.find(id); if (tournament_cache_iter != tournament_cache.end()) { @@ -411,8 +411,8 @@ private: } try { - object_id_type id = changed_object_variant["id"].as(); - match_object current_match_obj = changed_object_variant.as(); + object_id_type id = changed_object_variant["id"].as( GRAPHENE_MAX_NESTED_OBJECTS ); + match_object current_match_obj = changed_object_variant.as( GRAPHENE_MAX_NESTED_OBJECTS ); auto match_cache_iter = match_cache.find(id); if (match_cache_iter != match_cache.end()) { @@ -436,8 +436,8 @@ private: } try { - object_id_type id = changed_object_variant["id"].as(); - game_object current_game_obj = changed_object_variant.as(); + object_id_type id = changed_object_variant["id"].as( GRAPHENE_MAX_NESTED_OBJECTS ); + game_object current_game_obj = changed_object_variant.as( GRAPHENE_MAX_NESTED_OBJECTS ); auto game_cache_iter = game_cache.find(id); if (game_cache_iter != game_cache.end()) { @@ -460,10 +460,10 @@ private: } try { - object_id_type id = changed_object_variant["id"].as(); + object_id_type id = changed_object_variant["id"].as( GRAPHENE_MAX_NESTED_OBJECTS ); if (_wallet.my_accounts.find(id) != _wallet.my_accounts.end()) { - account_object account = changed_object_variant.as(); + account_object account = changed_object_variant.as( GRAPHENE_MAX_NESTED_OBJECTS ); _wallet.update_account(account); } continue; @@ -641,7 +641,7 @@ public: T get_object(object_id id)const { auto ob = _remote_db->get_objects({id}).front(); - return ob.template as(); + return ob.template as( GRAPHENE_MAX_NESTED_OBJECTS ); } void set_operation_fees( signed_transaction& tx, const fee_schedule& s ) @@ -657,16 +657,16 @@ public: auto dynamic_props = get_dynamic_global_properties(); fc::mutable_variant_object result; result["head_block_num"] = dynamic_props.head_block_number; - result["head_block_id"] = dynamic_props.head_block_id; + result["head_block_id"] = fc::variant(dynamic_props.head_block_id, 1); result["head_block_age"] = fc::get_approximate_relative_time_string(dynamic_props.time, time_point_sec(time_point::now()), " old"); result["next_maintenance_time"] = fc::get_approximate_relative_time_string(dynamic_props.next_maintenance_time); result["chain_id"] = chain_props.chain_id; result["participation"] = (100*dynamic_props.recent_slots_filled.popcount()) / 128.0; - result["active_witnesses"] = global_props.active_witnesses; - result["active_committee_members"] = global_props.active_committee_members; - result["entropy"] = dynamic_props.random; + result["active_witnesses"] = fc::variant(global_props.active_witnesses, GRAPHENE_MAX_NESTED_OBJECTS); + result["active_committee_members"] = fc::variant(global_props.active_committee_members, GRAPHENE_MAX_NESTED_OBJECTS); + result["entropy"] = fc::variant(dynamic_props.random, GRAPHENE_MAX_NESTED_OBJECTS); return result; } @@ -786,7 +786,7 @@ public: FC_ASSERT( asset_symbol_or_id.size() > 0 ); vector> opt_asset; if( std::isdigit( asset_symbol_or_id.front() ) ) - return fc::variant(asset_symbol_or_id).as(); + return fc::variant(asset_symbol_or_id, 1).as( 1 ); opt_asset = _remote_db->lookup_asset_symbols( {asset_symbol_or_id} ); FC_ASSERT( (opt_asset.size() > 0) && (opt_asset[0].valid()) ); return opt_asset[0]->id; @@ -998,7 +998,7 @@ public: if( ! fc::exists( wallet_filename ) ) return false; - _wallet = fc::json::from_file( wallet_filename ).as< wallet_data >(); + _wallet = fc::json::from_file( wallet_filename ).as< wallet_data >( 2 * GRAPHENE_MAX_NESTED_OBJECTS ); if( _wallet.chain_id != _chain_id ) FC_THROW( "Wallet chain ID does not match", ("wallet.chain_id", _wallet.chain_id) @@ -1828,7 +1828,6 @@ public: { try { witness_object witness = get_witness(witness_name); account_object witness_account = get_account( witness.witness_account ); - fc::ecc::private_key active_private_key = get_private_key_for_account(witness_account); witness_update_operation witness_update_op; witness_update_op.witness = witness.id; @@ -1852,7 +1851,7 @@ public: static WorkerInit _create_worker_initializer( const variant& worker_settings ) { WorkerInit result; - from_variant( worker_settings, result ); + from_variant( worker_settings, result, GRAPHENE_MAX_NESTED_OBJECTS ); return result; } @@ -1906,7 +1905,6 @@ public: ) { account_object acct = get_account( account ); - account_update_operation op; // you could probably use a faster algorithm for this, but flat_set is fast enough :) flat_set< worker_id_type > merged; @@ -1940,7 +1938,7 @@ public: for( const variant& obj : objects ) { worker_object wo; - from_variant( obj, wo ); + from_variant( obj, wo, GRAPHENE_MAX_NESTED_OBJECTS ); new_votes.erase( wo.vote_for ); new_votes.erase( wo.vote_against ); if( delta.vote_for.find( wo.id ) != delta.vote_for.end() ) @@ -2196,6 +2194,7 @@ public: owned_keys.reserve(pks.size()); std::copy_if(pks.begin(), pks.end(), std::inserter(owned_keys, owned_keys.end()), [this](const public_key_type &pk) { return _keys.find(pk) != _keys.end(); }); + tx.clear_signatures(); set approving_key_set = _remote_db->get_required_signatures(tx, owned_keys); auto dyn_props = get_dynamic_global_properties(); @@ -2213,8 +2212,8 @@ public: uint32_t expiration_time_offset = 0; for (;;) { - tx.set_expiration(dyn_props.time + fc::seconds(30 + expiration_time_offset)); - tx.signatures.clear(); + tx.set_expiration( dyn_props.time + fc::seconds(30 + expiration_time_offset) ); + tx.clear_signatures(); for (const public_key_type &key : approving_key_set) tx.sign(get_private_key(key), _chain_id); @@ -2295,7 +2294,6 @@ public: trx.operations = {op}; set_operation_fees( trx, _remote_db->get_global_properties().parameters.current_fees); trx.validate(); - idump((broadcast)); return sign_transaction(trx, broadcast); } @@ -2396,7 +2394,7 @@ public: m["get_account_history"] = [this](variant result, const fc::variants& a) { - auto r = result.as>(); + auto r = result.as>( GRAPHENE_MAX_NESTED_OBJECTS ); std::stringstream ss; for( operation_detail& d : r ) @@ -2413,7 +2411,7 @@ public: }; m["get_relative_account_history"] = [this](variant result, const fc::variants& a) { - auto r = result.as>(); + auto r = result.as>( GRAPHENE_MAX_NESTED_OBJECTS ); std::stringstream ss; for( operation_detail& d : r ) @@ -2431,7 +2429,7 @@ public: m["list_account_balances"] = [this](variant result, const fc::variants& a) { - auto r = result.as>(); + auto r = result.as>( GRAPHENE_MAX_NESTED_OBJECTS ); vector asset_recs; std::transform(r.begin(), r.end(), std::back_inserter(asset_recs), [this](const asset& a) { return get_asset(a.asset_id); @@ -2448,7 +2446,7 @@ public: { std::stringstream ss; - auto balances = result.as>(); + auto balances = result.as>( GRAPHENE_MAX_NESTED_OBJECTS ); for (const account_balance_object& balance: balances) { const account_object& account = get_account(balance.owner); @@ -2461,7 +2459,7 @@ public: m["get_blind_balances"] = [this](variant result, const fc::variants& a) { - auto r = result.as>(); + auto r = result.as>( GRAPHENE_MAX_NESTED_OBJECTS ); vector asset_recs; std::transform(r.begin(), r.end(), std::back_inserter(asset_recs), [this](const asset& a) { return get_asset(a.asset_id); @@ -2475,7 +2473,7 @@ public: }; m["transfer_to_blind"] = [this](variant result, const fc::variants& a) { - auto r = result.as(); + auto r = result.as( GRAPHENE_MAX_NESTED_OBJECTS ); std::stringstream ss; r.trx.operations[0].visit( operation_printer( ss, *this, operation_result() ) ); ss << "\n"; @@ -2488,7 +2486,7 @@ public: }; m["blind_transfer"] = [this](variant result, const fc::variants& a) { - auto r = result.as(); + auto r = result.as( GRAPHENE_MAX_NESTED_OBJECTS ); std::stringstream ss; r.trx.operations[0].visit( operation_printer( ss, *this, operation_result() ) ); ss << "\n"; @@ -2501,7 +2499,7 @@ public: }; m["receive_blind_transfer"] = [this](variant result, const fc::variants& a) { - auto r = result.as(); + auto r = result.as( GRAPHENE_MAX_NESTED_OBJECTS ); std::stringstream ss; asset_object as = get_asset( r.amount.asset_id ); ss << as.amount_to_pretty_string( r.amount ) << " " << r.from_label << " => " << r.to_label << " " << r.memo <<"\n"; @@ -2509,7 +2507,7 @@ public: }; m["blind_history"] = [this](variant result, const fc::variants& a) { - auto records = result.as>(); + auto records = result.as>( GRAPHENE_MAX_NESTED_OBJECTS ); std::stringstream ss; ss << "WHEN " << " " << "AMOUNT" << " " << "FROM" << " => " << "TO" << " " << "MEMO" <<"\n"; @@ -2524,14 +2522,14 @@ public: }; m["get_upcoming_tournaments"] = m["get_tournaments"] = m["get_tournaments_by_state"] = [this](variant result, const fc::variants& a) { - const vector tournaments = result.as >(); + const vector tournaments = result.as >( GRAPHENE_MAX_NESTED_OBJECTS ); std::stringstream ss; ss << "ID GAME BUY IN PLAYERS\n"; ss << "====================================================================================\n"; for( const tournament_object& tournament_obj : tournaments ) { asset_object buy_in_asset = get_asset(tournament_obj.options.buy_in.asset_id); - ss << fc::variant(tournament_obj.id).as() << " " + ss << fc::variant(tournament_obj.id, 1).as( 1 ) << " " << buy_in_asset.amount_to_pretty_string(tournament_obj.options.buy_in.amount) << " " << tournament_obj.options.number_of_players << " players\n"; switch (tournament_obj.get_state()) @@ -2574,8 +2572,8 @@ public: { std::stringstream ss; - tournament_object tournament = result.as(); - tournament_details_object tournament_details = _remote_db->get_objects({result["tournament_details_id"].as()})[0].as(); + tournament_object tournament = result.as( GRAPHENE_MAX_NESTED_OBJECTS ); + tournament_details_object tournament_details = _remote_db->get_objects({result["tournament_details_id"].as( 5 )})[0].as( 5 ); tournament_state state = tournament.get_state(); if (state == tournament_state::accepting_registrations) { @@ -2673,7 +2671,7 @@ public: }; m["get_order_book"] = [this](variant result, const fc::variants& a) { - auto orders = result.as(); + auto orders = result.as( GRAPHENE_MAX_NESTED_OBJECTS ); auto bids = orders.bids; auto asks = orders.asks; std::stringstream ss; @@ -2683,12 +2681,10 @@ public: double ask_sum = 0; const int spacing = 20; - auto prettify_num = [&]( double n ) + auto prettify_num = [&ss]( double n ) { - //ss << n; if (abs( round( n ) - n ) < 0.00000000001 ) { - //ss << setiosflags( !ios::fixed ) << (int) n; // doesn't compile on Linux with gcc ss << (int) n; } else if (n - floor(n) < 0.000001) @@ -2770,7 +2766,7 @@ public: const chain_parameters& current_params = get_global_properties().parameters; chain_parameters new_params = current_params; fc::reflector::visit( - fc::from_variant_visitor( changed_values, new_params ) + fc::from_variant_visitor( changed_values, new_params, GRAPHENE_MAX_NESTED_OBJECTS ) ); committee_member_update_global_parameters_operation update_op; @@ -2820,7 +2816,7 @@ public: continue; } // is key a number? - auto is_numeric = [&]() -> bool + auto is_numeric = [&key]() -> bool { size_t n = key.size(); for( size_t i=0; isecond; } - fee_parameters fp = from_which_variant< fee_parameters >( which, item.value() ); + fee_parameters fp = from_which_variant< fee_parameters >( which, item.value(), GRAPHENE_MAX_NESTED_OBJECTS ); fee_map[ which ] = fp; } @@ -2886,7 +2882,7 @@ public: const chain_parameters& current_params = get_global_properties().parameters; asset_update_dividend_operation changed_op; fc::reflector::visit( - fc::from_variant_visitor( changed_values, changed_op ) + fc::from_variant_visitor( changed_values, changed_op, GRAPHENE_MAX_NESTED_OBJECTS ) ); optional asset_to_update = find_asset(changed_op.asset_to_update); @@ -2924,7 +2920,7 @@ public: proposal_update_operation update_op; update_op.fee_paying_account = get_account(fee_paying_account).id; - update_op.proposal = fc::variant(proposal_id).as(); + update_op.proposal = fc::variant(proposal_id, 1).as( 1 ); // make sure the proposal exists get_object( update_op.proposal ); @@ -3051,7 +3047,7 @@ public: for( const auto& peer : peers ) { variant v; - fc::to_variant( peer, v ); + fc::to_variant( peer, v, GRAPHENE_MAX_NESTED_OBJECTS ); result.push_back( v ); } return result; @@ -3064,7 +3060,6 @@ public: const account_object& master = *_wallet.my_accounts.get().lower_bound("import"); int number_of_accounts = number_of_transactions / 3; number_of_transactions -= number_of_accounts; - //auto key = derive_private_key("floodshill", 0); try { dbg_make_uia(master.name, "SHILL"); } catch(...) {/* Ignore; the asset probably already exists.*/} @@ -3429,6 +3424,11 @@ vector wallet_api::list_assets(const string& lowerbound, uint32_t return my->_remote_db->list_assets( lowerbound, limit ); } +uint64_t wallet_api::get_asset_count()const +{ + return my->_remote_db->get_asset_count(); +} + vector wallet_api::get_lotteries( asset_id_type stop, unsigned limit, asset_id_type start )const @@ -3449,30 +3449,54 @@ asset wallet_api::get_lottery_balance( asset_id_type lottery_id )const return my->_remote_db->get_lottery_balance( lottery_id ); } -vector wallet_api::get_account_history(string name, int limit)const +vector wallet_api::get_account_history(string name, int limit) const { vector result; auto account_id = get_account(name).get_id(); - while( limit > 0 ) + while (limit > 0) { + bool skip_first_row = false; operation_history_id_type start; - if( result.size() ) + if (result.size()) { start = result.back().op.id; - start = start + 1; + if (start == operation_history_id_type()) // no more data + break; + start = start + (-1); + if (start == operation_history_id_type()) // will return most recent history if directly call remote API with this + { + start = start + 1; + skip_first_row = true; + } } + int page_limit = skip_first_row ? std::min(100, limit + 1) : std::min(100, limit); - vector current = my->_remote_hist->get_account_history(account_id, operation_history_id_type(), std::min(100,limit), start); - for( auto& o : current ) { + vector current = my->_remote_hist->get_account_history(account_id, operation_history_id_type(), + page_limit, start); + bool first_row = true; + for (auto &o : current) + { + if (first_row) + { + first_row = false; + if (skip_first_row) + { + continue; + } + } std::stringstream ss; auto memo = o.op.visit(detail::operation_printer(ss, *my, o.result)); - result.push_back( operation_detail{ memo, ss.str(), o } ); + result.push_back(operation_detail{memo, ss.str(), o}); } - if( (int)current.size() < std::min(100,limit) ) + + if (int(current.size()) < page_limit) break; + limit -= current.size(); + if (skip_first_row) + ++limit; } return result; @@ -4532,7 +4556,7 @@ string wallet_api::get_private_key( public_key_type pubkey )const public_key_type wallet_api::get_public_key( string label )const { - try { return fc::variant(label).as(); } catch ( ... ){} + try { return fc::variant(label, 1).as( 1 ); } catch ( ... ){} auto key_itr = my->_wallet.labeled_keys.get().find(label); if( key_itr != my->_wallet.labeled_keys.get().end() ) @@ -5710,7 +5734,7 @@ vector wallet_api::get_tournaments_by_state(tournament_id_typ tournament_object wallet_api::get_tournament(tournament_id_type id) { - return my->_remote_db->get_objects({id})[0].as(); + return my->_remote_db->get_objects({id})[0].as( GRAPHENE_MAX_NESTED_OBJECTS ); } signed_transaction wallet_api::rps_throw(game_id_type game_id, @@ -5822,13 +5846,15 @@ vesting_balance_object_with_info::vesting_balance_object_with_info( const vestin } } // graphene::wallet -void fc::to_variant(const account_multi_index_type& accts, fc::variant& vo) -{ - vo = vector(accts.begin(), accts.end()); -} +namespace fc { + void to_variant( const account_multi_index_type& accts, variant& vo, uint32_t max_depth ) + { + to_variant( std::vector(accts.begin(), accts.end()), vo, max_depth ); + } -void fc::from_variant(const fc::variant& var, account_multi_index_type& vo) -{ - const vector& v = var.as>(); - vo = account_multi_index_type(v.begin(), v.end()); + void from_variant( const variant& var, account_multi_index_type& vo, uint32_t max_depth ) + { + const std::vector& v = var.as>( max_depth ); + vo = account_multi_index_type(v.begin(), v.end()); + } } diff --git a/programs/build_helpers/cat-parts b/programs/build_helpers/cat-parts index 592619b2..2bcd1c8a 100755 Binary files a/programs/build_helpers/cat-parts and b/programs/build_helpers/cat-parts differ diff --git a/programs/build_helpers/member_enumerator.cpp b/programs/build_helpers/member_enumerator.cpp index 8ad26633..915d7edf 100644 --- a/programs/build_helpers/member_enumerator.cpp +++ b/programs/build_helpers/member_enumerator.cpp @@ -37,7 +37,7 @@ namespace graphene { namespace member_enumerator { struct class_processor { - class_processor( std::map< std::string, std::vector< std::string > >& r ) : result(r) {} + explicit class_processor( std::map< std::string, std::vector< std::string > >& r ) : result(r) {} template< typename T > void process_class( const T* dummy ); @@ -84,7 +84,7 @@ struct member_visitor struct static_variant_visitor { - static_variant_visitor( class_processor* p ) : proc(p) {} + explicit static_variant_visitor( class_processor* p ) : proc(p) {} typedef void result_type; @@ -215,13 +215,12 @@ int main( int argc, char** argv ) { std::map< std::string, std::vector< std::string > > result; graphene::member_enumerator::class_processor::process_class(result); - //graphene::member_enumerator::process_class(result); fc::mutable_variant_object mvo; for( const std::pair< std::string, std::vector< std::string > >& e : result ) { variant v; - to_variant( e.second, v ); + to_variant( e.second, v , 1); mvo.set( e.first, v ); } diff --git a/programs/cli_wallet/main.cpp b/programs/cli_wallet/main.cpp index 53404633..d68f25b8 100644 --- a/programs/cli_wallet/main.cpp +++ b/programs/cli_wallet/main.cpp @@ -37,6 +37,7 @@ #include #include +#include #include #include #include @@ -108,8 +109,8 @@ int main( int argc, char** argv ) std::cout << "Logging RPC to file: " << (data_dir / ac.filename).preferred_string() << "\n"; - cfg.appenders.push_back(fc::appender_config( "default", "console", fc::variant(fc::console_appender::config()))); - cfg.appenders.push_back(fc::appender_config( "rpc", "file", fc::variant(ac))); + cfg.appenders.push_back(fc::appender_config( "default", "console", fc::variant(fc::console_appender::config(), 20))); + cfg.appenders.push_back(fc::appender_config( "rpc", "file", fc::variant(ac, 5))); cfg.loggers = { fc::logger_config("default"), fc::logger_config( "rpc") }; cfg.loggers.front().level = fc::log_level::info; @@ -117,8 +118,6 @@ int main( int argc, char** argv ) cfg.loggers.back().level = fc::log_level::debug; cfg.loggers.back().appenders = {"rpc"}; - //fc::configure_logging( cfg ); - fc::ecc::private_key committee_private_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key"))); idump( (key_to_wif( committee_private_key ) ) ); @@ -139,7 +138,7 @@ int main( int argc, char** argv ) fc::path wallet_file( options.count("wallet-file") ? options.at("wallet-file").as() : "wallet.json"); if( fc::exists( wallet_file ) ) { - wdata = fc::json::from_file( wallet_file ).as(); + wdata = fc::json::from_file( wallet_file ).as( GRAPHENE_MAX_NESTED_OBJECTS ); if( options.count("chain-id") ) { // the --chain-id on the CLI must match the chain ID embedded in the wallet file @@ -175,12 +174,11 @@ int main( int argc, char** argv ) fc::http::websocket_client client; idump((wdata.ws_server)); auto con = client.connect( wdata.ws_server ); - auto apic = std::make_shared(con); + auto apic = std::make_shared(con, GRAPHENE_MAX_NESTED_OBJECTS); auto remote_api = apic->get_remote_api< login_api >(1); edump((wdata.ws_user)(wdata.ws_password) ); - // TODO: Error message here - FC_ASSERT( remote_api->login( wdata.ws_user, wdata.ws_password ) ); + FC_ASSERT( remote_api->login( wdata.ws_user, wdata.ws_password ), "Failed to log in to API server" ); auto wapiptr = std::make_shared( wdata, remote_api ); wapiptr->set_wallet_filename( wallet_file.generic_string() ); @@ -188,11 +186,11 @@ int main( int argc, char** argv ) fc::api wapi(wapiptr); - auto wallet_cli = std::make_shared(); + auto wallet_cli = std::make_shared( GRAPHENE_MAX_NESTED_OBJECTS ); for( auto& name_formatter : wapiptr->get_result_formatters() ) wallet_cli->format_result( name_formatter.first, name_formatter.second ); - boost::signals2::scoped_connection closed_connection(con->closed.connect([=]{ + boost::signals2::scoped_connection closed_connection(con->closed.connect([wallet_cli]{ cerr << "Server has disconnected us.\n"; wallet_cli->stop(); })); @@ -212,10 +210,10 @@ int main( int argc, char** argv ) auto _websocket_server = std::make_shared(); if( options.count("rpc-endpoint") ) { - _websocket_server->on_connection([&]( const fc::http::websocket_connection_ptr& c ){ + _websocket_server->on_connection([&wapi]( const fc::http::websocket_connection_ptr& c ){ std::cout << "here... \n"; wlog("." ); - auto wsc = std::make_shared(c); + auto wsc = std::make_shared(c, GRAPHENE_MAX_NESTED_OBJECTS); wsc->register_api(wapi); c->set_session_data( wsc ); }); @@ -232,7 +230,7 @@ int main( int argc, char** argv ) if( options.count("rpc-tls-endpoint") ) { _websocket_tls_server->on_connection([&]( const fc::http::websocket_connection_ptr& c ){ - auto wsc = std::make_shared(c); + auto wsc = std::make_shared(c, GRAPHENE_MAX_NESTED_OBJECTS); wsc->register_api(wapi); c->set_session_data( wsc ); }); @@ -250,10 +248,10 @@ int main( int argc, char** argv ) // due to implementation, on_request() must come AFTER listen() // _http_server->on_request( - [&]( const fc::http::request& req, const fc::http::server::response& resp ) + [&wapi]( const fc::http::request& req, const fc::http::server::response& resp ) { std::shared_ptr< fc::rpc::http_api_connection > conn = - std::make_shared< fc::rpc::http_api_connection>(); + std::make_shared< fc::rpc::http_api_connection >( GRAPHENE_MAX_NESTED_OBJECTS ); conn->register_api( wapi ); conn->on_request( req, resp ); } ); diff --git a/programs/debug_node/main.cpp b/programs/debug_node/main.cpp index 7c3d358a..c94d3fd2 100644 --- a/programs/debug_node/main.cpp +++ b/programs/debug_node/main.cpp @@ -261,8 +261,8 @@ fc::optional load_logging_config_from_ini_file(const fc::pat console_appender_config.level_colors.emplace_back( fc::console_appender::level_color(fc::log_level::error, fc::console_appender::color::cyan)); - console_appender_config.stream = fc::variant(stream_name).as(); - logging_config.appenders.push_back(fc::appender_config(console_appender_name, "console", fc::variant(console_appender_config))); + console_appender_config.stream = fc::variant(stream_name, 1).as(1); + logging_config.appenders.push_back(fc::appender_config(console_appender_name, "console", fc::variant(console_appender_config, 20))); found_logging_config = true; } else if (boost::starts_with(section_name, file_appender_section_prefix)) @@ -281,7 +281,7 @@ fc::optional load_logging_config_from_ini_file(const fc::pat file_appender_config.rotate = true; file_appender_config.rotation_interval = fc::hours(1); file_appender_config.rotation_limit = fc::days(1); - logging_config.appenders.push_back(fc::appender_config(file_appender_name, "file", fc::variant(file_appender_config))); + logging_config.appenders.push_back(fc::appender_config(file_appender_name, "file", fc::variant(file_appender_config, 20))); found_logging_config = true; } else if (boost::starts_with(section_name, logger_section_prefix)) @@ -290,7 +290,7 @@ fc::optional load_logging_config_from_ini_file(const fc::pat std::string level_string = section_tree.get("level"); std::string appenders_string = section_tree.get("appenders"); fc::logger_config logger_config(logger_name); - logger_config.level = fc::variant(level_string).as(); + logger_config.level = fc::variant(level_string, 1).as(1); boost::split(logger_config.appenders, appenders_string, boost::is_any_of(" ,"), boost::token_compress_on); diff --git a/programs/delayed_node/main.cpp b/programs/delayed_node/main.cpp index 44d12f37..5894dcf8 100644 --- a/programs/delayed_node/main.cpp +++ b/programs/delayed_node/main.cpp @@ -258,8 +258,8 @@ fc::optional load_logging_config_from_ini_file(const fc::pat console_appender_config.level_colors.emplace_back( fc::console_appender::level_color(fc::log_level::error, fc::console_appender::color::cyan)); - console_appender_config.stream = fc::variant(stream_name).as(); - logging_config.appenders.push_back(fc::appender_config(console_appender_name, "console", fc::variant(console_appender_config))); + console_appender_config.stream = fc::variant(stream_name, 1).as(1); + logging_config.appenders.push_back(fc::appender_config(console_appender_name, "console", fc::variant(console_appender_config, GRAPHENE_MAX_NESTED_OBJECTS))); found_logging_config = true; } else if (boost::starts_with(section_name, file_appender_section_prefix)) @@ -278,7 +278,7 @@ fc::optional load_logging_config_from_ini_file(const fc::pat file_appender_config.rotate = true; file_appender_config.rotation_interval = fc::hours(1); file_appender_config.rotation_limit = fc::days(1); - logging_config.appenders.push_back(fc::appender_config(file_appender_name, "file", fc::variant(file_appender_config))); + logging_config.appenders.push_back(fc::appender_config(file_appender_name, "file", fc::variant(file_appender_config, GRAPHENE_MAX_NESTED_OBJECTS))); found_logging_config = true; } else if (boost::starts_with(section_name, logger_section_prefix)) @@ -287,7 +287,7 @@ fc::optional load_logging_config_from_ini_file(const fc::pat std::string level_string = section_tree.get("level"); std::string appenders_string = section_tree.get("appenders"); fc::logger_config logger_config(logger_name); - logger_config.level = fc::variant(level_string).as(); + logger_config.level = fc::variant(level_string, 1).as(1); boost::split(logger_config.appenders, appenders_string, boost::is_any_of(" ,"), boost::token_compress_on); diff --git a/programs/genesis_util/genesis_update.cpp b/programs/genesis_util/genesis_update.cpp index 52329301..e753b8b7 100644 --- a/programs/genesis_util/genesis_update.cpp +++ b/programs/genesis_util/genesis_update.cpp @@ -110,7 +110,7 @@ int main( int argc, char** argv ) std::cerr << "update_genesis: Reading genesis from file " << genesis_json_filename.preferred_string() << "\n"; std::string genesis_json; read_file_contents( genesis_json_filename, genesis_json ); - genesis = fc::json::from_string( genesis_json ).as< genesis_state_type >(); + genesis = fc::json::from_string( genesis_json ).as< genesis_state_type >(20); } else { @@ -120,8 +120,8 @@ int main( int argc, char** argv ) if (!options.count("nop")) { - std::string dev_key_prefix = options["dev-key-prefix"].as(); - auto get_dev_key = [&]( std::string prefix, uint32_t i ) -> public_key_type + const std::string dev_key_prefix = options["dev-key-prefix"].as(); + auto get_dev_key = [&dev_key_prefix]( std::string prefix, uint32_t i ) -> public_key_type { return fc::ecc::private_key::regenerate( fc::sha256::hash( dev_key_prefix + prefix + std::to_string(i) ) ).get_public_key(); }; diff --git a/programs/genesis_util/get_dev_key.cpp b/programs/genesis_util/get_dev_key.cpp index c82e6a60..ea7cdf9f 100644 --- a/programs/genesis_util/get_dev_key.cpp +++ b/programs/genesis_util/get_dev_key.cpp @@ -70,9 +70,9 @@ int main( int argc, char** argv ) bool comma = false; - auto show_key = [&]( const fc::ecc::private_key& priv_key ) + auto show_key = [&comma]( const fc::ecc::private_key& priv_key ) { - fc::mutable_variant_object mvo; + fc::limited_mutable_variant_object mvo(5); graphene::chain::public_key_type pub_key = priv_key.get_public_key(); mvo( "private_key", graphene::utilities::key_to_wif( priv_key ) ) ( "public_key", std::string( pub_key ) ) @@ -80,7 +80,7 @@ int main( int argc, char** argv ) ; if( comma ) std::cout << ",\n"; - std::cout << fc::json::to_string( mvo ); + std::cout << fc::json::to_string( fc::mutable_variant_object(mvo) ); comma = true; }; @@ -90,7 +90,7 @@ int main( int argc, char** argv ) { std::string arg = argv[i]; std::string prefix; - int lep = -1, rep; + int lep = -1, rep = -1; auto dash_pos = arg.rfind('-'); if( dash_pos != string::npos ) { @@ -104,7 +104,6 @@ int main( int argc, char** argv ) rep = std::stoi( rhs.substr( colon_pos+1 ) ); } } - vector< fc::ecc::private_key > keys; if( lep >= 0 ) { for( int k=lep; k(); boost::program_options::variables_map cfg; - cfg.emplace("p2p-endpoint", boost::program_options::variable_value(string("127.0.0.1:3939"), false)); + cfg.emplace("p2p-endpoint", boost::program_options::variable_value(string("127.0.0.1:0"), false)); app1.initialize(app_dir.path(), cfg); + cfg.emplace("genesis-json", boost::program_options::variable_value(create_genesis_file(app_dir), false)); + + BOOST_TEST_MESSAGE( "Starting app1 and waiting 1500 ms" ); + app1.startup(); + fc::usleep(fc::milliseconds(500)); + string endpoint1 = app1.p2p_node()->get_actual_listening_endpoint(); BOOST_TEST_MESSAGE( "Creating and initializing app2" ); + auto cfg2 = cfg; graphene::app::application app2; app2.register_plugin(); - auto cfg2 = cfg; cfg2.erase("p2p-endpoint"); - cfg2.emplace("p2p-endpoint", boost::program_options::variable_value(string("127.0.0.1:4040"), false)); - cfg2.emplace("seed-node", boost::program_options::variable_value(vector{"127.0.0.1:3939"}, false)); + cfg2.emplace("p2p-endpoint", boost::program_options::variable_value(string("127.0.0.1:0"), false)); + cfg2.emplace("seed-node", boost::program_options::variable_value(vector{endpoint1}, false)); app2.initialize(app2_dir.path(), cfg2); - - cfg.emplace("genesis-json", boost::program_options::variable_value(create_genesis_file(app_dir), false)); cfg2.emplace("genesis-json", boost::program_options::variable_value(create_genesis_file(app2_dir), false)); - BOOST_TEST_MESSAGE( "Starting app1 and waiting 1500 ms" ); - app1.startup(); - fc::usleep(fc::milliseconds(1500)); BOOST_TEST_MESSAGE( "Starting app2 and waiting 1500 ms" ); app2.startup(); - fc::usleep(fc::milliseconds(1500)); + int counter = 0; + while(!app2.p2p_node()->is_connected()) + { + fc::usleep(fc::milliseconds(500)); + if(counter++ >= 100) + break; + } BOOST_REQUIRE_EQUAL(app1.p2p_node()->get_connection_count(), 1); BOOST_CHECK_EQUAL(std::string(app1.p2p_node()->get_connected_peers().front().host.get_address()), "127.0.0.1"); diff --git a/tests/benchmarks/genesis_allocation.cpp b/tests/benchmarks/genesis_allocation.cpp index 61a3b1b8..a17a16fa 100644 --- a/tests/benchmarks/genesis_allocation.cpp +++ b/tests/benchmarks/genesis_allocation.cpp @@ -68,7 +68,7 @@ BOOST_AUTO_TEST_CASE( genesis_and_persistence_bench ) { database db; - db.open(data_dir.path(), [&]{return genesis_state;}); + db.open(data_dir.path(), [&]{return genesis_state;}, "test"); for( int i = 11; i < account_count + 11; ++i) BOOST_CHECK(db.get_balance(account_id_type(i), asset_id_type()).amount == GRAPHENE_MAX_SHARE_SUPPLY / account_count); @@ -81,7 +81,7 @@ BOOST_AUTO_TEST_CASE( genesis_and_persistence_bench ) database db; fc::time_point start_time = fc::time_point::now(); - db.open(data_dir.path(), [&]{return genesis_state;}); + db.open(data_dir.path(), [&]{return genesis_state;}, "test"); ilog("Opened database in ${t} milliseconds.", ("t", (fc::time_point::now() - start_time).count() / 1000)); for( int i = 11; i < account_count + 11; ++i) @@ -116,7 +116,7 @@ BOOST_AUTO_TEST_CASE( genesis_and_persistence_bench ) auto start_time = fc::time_point::now(); wlog( "about to start reindex..." ); - db.reindex(data_dir.path(), genesis_state); + db.open(data_dir.path(), [&]{return genesis_state;}, "force_wipe"); ilog("Replayed database in ${t} milliseconds.", ("t", (fc::time_point::now() - start_time).count() / 1000)); for( int i = 0; i < blocks_to_produce; ++i ) diff --git a/tests/betting/betting_tests.cpp b/tests/betting/betting_tests.cpp index 33cc4fad..4befe25c 100644 --- a/tests/betting/betting_tests.cpp +++ b/tests/betting/betting_tests.cpp @@ -962,7 +962,7 @@ BOOST_AUTO_TEST_CASE(persistent_objects_test) fc::variants objects_from_bookie = bookie_api.get_objects({automatically_canceled_bet_id}); idump((objects_from_bookie)); BOOST_REQUIRE_EQUAL(objects_from_bookie.size(), 1u); - BOOST_CHECK_MESSAGE(objects_from_bookie[0]["id"].as() == automatically_canceled_bet_id, "Bookie Plugin didn't return a deleted bet it"); + BOOST_CHECK_MESSAGE(objects_from_bookie[0]["id"].as(1) == automatically_canceled_bet_id, "Bookie Plugin didn't return a deleted bet it"); // lay 47 at 1.94 odds (50:47) -- this bet should go on the order books normally bet_id_type first_bet_on_books = place_bet(alice_id, capitals_win_market.id, bet_type::lay, asset(47, asset_id_type()), 194 * GRAPHENE_BETTING_ODDS_PRECISION / 100); @@ -971,7 +971,7 @@ BOOST_AUTO_TEST_CASE(persistent_objects_test) objects_from_bookie = bookie_api.get_objects({first_bet_on_books}); idump((objects_from_bookie)); BOOST_REQUIRE_EQUAL(objects_from_bookie.size(), 1u); - BOOST_CHECK_MESSAGE(objects_from_bookie[0]["id"].as() == first_bet_on_books, "Bookie Plugin didn't return a bet that is currently on the books"); + BOOST_CHECK_MESSAGE(objects_from_bookie[0]["id"].as(1) == first_bet_on_books, "Bookie Plugin didn't return a bet that is currently on the books"); // place a bet that exactly matches 'first_bet_on_books', should result in empty books (thus, no bet_objects from the blockchain) bet_id_type matching_bet = place_bet(bob_id, capitals_win_market.id, bet_type::back, asset(50, asset_id_type()), 194 * GRAPHENE_BETTING_ODDS_PRECISION / 100); @@ -982,8 +982,8 @@ BOOST_AUTO_TEST_CASE(persistent_objects_test) objects_from_bookie = bookie_api.get_objects({first_bet_on_books, matching_bet}); idump((objects_from_bookie)); BOOST_REQUIRE_EQUAL(objects_from_bookie.size(), 2u); - BOOST_CHECK_MESSAGE(objects_from_bookie[0]["id"].as() == first_bet_on_books, "Bookie Plugin didn't return a bet that has been filled"); - BOOST_CHECK_MESSAGE(objects_from_bookie[1]["id"].as() == matching_bet, "Bookie Plugin didn't return a bet that has been filled"); + BOOST_CHECK_MESSAGE(objects_from_bookie[0]["id"].as(1) == first_bet_on_books, "Bookie Plugin didn't return a bet that has been filled"); + BOOST_CHECK_MESSAGE(objects_from_bookie[1]["id"].as(1) == matching_bet, "Bookie Plugin didn't return a bet that has been filled"); update_betting_market_group(moneyline_betting_markets.id, _status = betting_market_group_status::closed); @@ -1249,7 +1249,7 @@ BOOST_AUTO_TEST_CASE( chained_market_create_test ) for (const witness_id_type& witness_id : active_witnesses) { - BOOST_TEST_MESSAGE("Approving sport+competitors creation from witness " << fc::variant(witness_id).as()); + BOOST_TEST_MESSAGE("Approving sport+competitors creation from witness " << fc::variant(witness_id, 1).as(1)); const witness_object& witness = witness_id(db); const account_object& witness_account = witness.witness_account(db); @@ -2077,7 +2077,7 @@ BOOST_AUTO_TEST_CASE(event_driven_standard_progression_1) // removed. fc::variants objects_from_bookie = bookie_api.get_objects({capitals_vs_blackhawks_id}); - BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(), "settled"); + BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(1), "settled"); } FC_LOG_AND_RETHROW() } @@ -2138,12 +2138,12 @@ BOOST_AUTO_TEST_CASE(event_driven_standard_progression_1_with_delay) blackhawks_win_market_id}); idump((objects_from_bookie)); - BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(), "settled"); - BOOST_CHECK_EQUAL(objects_from_bookie[1]["status"].as(), "settled"); - BOOST_CHECK_EQUAL(objects_from_bookie[2]["status"].as(), "settled"); - BOOST_CHECK_EQUAL(objects_from_bookie[2]["resolution"].as(), "win"); - BOOST_CHECK_EQUAL(objects_from_bookie[3]["status"].as(), "settled"); - BOOST_CHECK_EQUAL(objects_from_bookie[3]["resolution"].as(), "not_win"); + BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(1), "settled"); + BOOST_CHECK_EQUAL(objects_from_bookie[1]["status"].as(1), "settled"); + BOOST_CHECK_EQUAL(objects_from_bookie[2]["status"].as(1), "settled"); + BOOST_CHECK_EQUAL(objects_from_bookie[2]["resolution"].as(1), "win"); + BOOST_CHECK_EQUAL(objects_from_bookie[3]["status"].as(1), "settled"); + BOOST_CHECK_EQUAL(objects_from_bookie[3]["resolution"].as(1), "not_win"); } FC_LOG_AND_RETHROW() } @@ -2230,7 +2230,7 @@ BOOST_AUTO_TEST_CASE(event_driven_standard_progression_2) // removed. fc::variants objects_from_bookie = bookie_api.get_objects({capitals_vs_blackhawks_id}); - BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(), "settled"); + BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(1), "settled"); } FC_LOG_AND_RETHROW() } @@ -2318,7 +2318,7 @@ BOOST_AUTO_TEST_CASE(event_driven_standard_progression_2_never_in_play) // removed. fc::variants objects_from_bookie = bookie_api.get_objects({capitals_vs_blackhawks_id}); - BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(), "settled"); + BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(1), "settled"); } FC_LOG_AND_RETHROW() } @@ -2393,7 +2393,7 @@ BOOST_AUTO_TEST_CASE(event_driven_standard_progression_3) // and group will cease to exist. The event should transition to "canceled", then be removed fc::variants objects_from_bookie = bookie_api.get_objects({capitals_vs_blackhawks_id}); - BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(), "canceled"); + BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(1), "canceled"); } FC_LOG_AND_RETHROW() } @@ -2488,7 +2488,7 @@ BOOST_AUTO_TEST_CASE(event_driven_progression_errors_1) generate_blocks(1); fc::variants objects_from_bookie = bookie_api.get_objects({capitals_vs_blackhawks_id}); - BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(), "canceled"); + BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(1), "canceled"); // we can't go back to upcoming, in_progress, frozen, or finished once we're canceled. // (this won't work because the event has been deleted) @@ -2540,7 +2540,7 @@ BOOST_AUTO_TEST_CASE(event_driven_progression_errors_2) // as soon as a block is generated, the betting market group will settle, and the market // and group will cease to exist. The event should transition to "settled", then removed fc::variants objects_from_bookie = bookie_api.get_objects({capitals_vs_blackhawks_id}); - BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(), "settled"); + BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(1), "settled"); // we can't go back to upcoming, in_progress, frozen, or finished once we're canceled. // (this won't work because the event has been deleted) @@ -2612,7 +2612,7 @@ BOOST_AUTO_TEST_CASE(betting_market_group_driven_standard_progression) // as soon as a block is generated, the betting market group will settle, and the market // and group will cease to exist. The event should transition to "settled" fc::variants objects_from_bookie = bookie_api.get_objects({capitals_vs_blackhawks_id}); - BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(), "settled"); + BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(1), "settled"); } FC_LOG_AND_RETHROW() } @@ -2723,7 +2723,7 @@ BOOST_AUTO_TEST_CASE(multi_betting_market_group_driven_standard_progression) // as soon as a block is generated, the two betting market groups will settle, and the market // and group will cease to exist. The event should transition to "settled" fc::variants objects_from_bookie = bookie_api.get_objects({capitals_vs_blackhawks_id}); - BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(), "settled"); + BOOST_CHECK_EQUAL(objects_from_bookie[0]["status"].as(1), "settled"); } FC_LOG_AND_RETHROW() } @@ -2834,13 +2834,13 @@ BOOST_AUTO_TEST_CASE( wimbledon_2017_gentelmen_singles_sf_test ) transfer(account_id_type(), alice_id, asset(10000000)); transfer(account_id_type(), bob_id, asset(10000000)); - BOOST_TEST_MESSAGE("moneyline_berdych_vs_federer " << fc::variant(moneyline_berdych_vs_federer.id).as()); - BOOST_TEST_MESSAGE("moneyline_cilic_vs_querrey " << fc::variant(moneyline_cilic_vs_querrey.id).as()); + BOOST_TEST_MESSAGE("moneyline_berdych_vs_federer " << fc::variant(moneyline_berdych_vs_federer.id, 1).as(1)); + BOOST_TEST_MESSAGE("moneyline_cilic_vs_querrey " << fc::variant(moneyline_cilic_vs_querrey.id, 1).as(1)); - BOOST_TEST_MESSAGE("berdych_wins_market " << fc::variant(berdych_wins_market.id).as()); - BOOST_TEST_MESSAGE("federer_wins_market " << fc::variant(federer_wins_market.id).as()); - BOOST_TEST_MESSAGE("cilic_wins_market " << fc::variant(cilic_wins_market.id).as()); - BOOST_TEST_MESSAGE("querrey_wins_market " << fc::variant(querrey_wins_market.id).as()); + BOOST_TEST_MESSAGE("berdych_wins_market " << fc::variant(berdych_wins_market.id, 1).as(1)); + BOOST_TEST_MESSAGE("federer_wins_market " << fc::variant(federer_wins_market.id, 1).as(1)); + BOOST_TEST_MESSAGE("cilic_wins_market " << fc::variant(cilic_wins_market.id, 1).as(1)); + BOOST_TEST_MESSAGE("querrey_wins_market " << fc::variant(querrey_wins_market.id, 1).as(1)); place_bet(alice_id, berdych_wins_market.id, bet_type::back, asset(1000000, asset_id_type()), 2 * GRAPHENE_BETTING_ODDS_PRECISION); place_bet(bob_id, berdych_wins_market.id, bet_type::lay, asset(1000000, asset_id_type()), 2 * GRAPHENE_BETTING_ODDS_PRECISION); @@ -2895,10 +2895,10 @@ BOOST_AUTO_TEST_CASE( wimbledon_2017_gentelmen_singles_final_test ) transfer(account_id_type(), alice_id, asset(10000000)); transfer(account_id_type(), bob_id, asset(10000000)); - BOOST_TEST_MESSAGE("moneyline_cilic_vs_federer " << fc::variant(moneyline_cilic_vs_federer.id).as()); + BOOST_TEST_MESSAGE("moneyline_cilic_vs_federer " << fc::variant(moneyline_cilic_vs_federer.id, 1).as(1)); - BOOST_TEST_MESSAGE("federer_wins_final_market " << fc::variant(federer_wins_final_market.id).as()); - BOOST_TEST_MESSAGE("cilic_wins_final_market " << fc::variant(cilic_wins_final_market.id).as()); + BOOST_TEST_MESSAGE("federer_wins_final_market " << fc::variant(federer_wins_final_market.id, 1).as(1)); + BOOST_TEST_MESSAGE("cilic_wins_final_market " << fc::variant(cilic_wins_final_market.id, 1).as(1)); betting_market_group_id_type moneyline_cilic_vs_federer_id = moneyline_cilic_vs_federer.id; update_betting_market_group(moneyline_cilic_vs_federer_id, _status = betting_market_group_status::in_play); diff --git a/tests/cli/main.cpp b/tests/cli/main.cpp index 82adb1c5..ee59f40f 100644 --- a/tests/cli/main.cpp +++ b/tests/cli/main.cpp @@ -27,8 +27,10 @@ #include #include +#include #include #include +#include #include #include @@ -117,6 +119,7 @@ std::shared_ptr start_application(fc::temp_directory std::shared_ptr app1(new graphene::app::application{}); app1->register_plugin< graphene::bookie::bookie_plugin>(); + app1->register_plugin(); app1->startup_plugins(); boost::program_options::variables_map cfg; #ifdef _WIN32 @@ -211,7 +214,7 @@ public: wallet_data.ws_password = ""; websocket_connection = websocket_client.connect( wallet_data.ws_server ); - api_connection = std::make_shared(websocket_connection); + api_connection = std::make_shared(websocket_connection, GRAPHENE_MAX_NESTED_OBJECTS); remote_login_api = api_connection->get_remote_api< graphene::app::login_api >(1); BOOST_CHECK(remote_login_api->login( wallet_data.ws_user, wallet_data.ws_password ) ); @@ -222,7 +225,7 @@ public: wallet_api = fc::api(wallet_api_ptr); - wallet_cli = std::make_shared(); + wallet_cli = std::make_shared(GRAPHENE_MAX_NESTED_OBJECTS); for( auto& name_formatter : wallet_api_ptr->get_result_formatters() ) wallet_cli->format_result( name_formatter.first, name_formatter.second ); @@ -438,3 +441,42 @@ BOOST_FIXTURE_TEST_CASE( cli_vote_for_2_witnesses, cli_fixture ) throw; } } + +/////////////////////// +// Check account history pagination +/////////////////////// +BOOST_FIXTURE_TEST_CASE( account_history_pagination, cli_fixture ) +{ + try + { + INVOKE(create_new_account); + + // attempt to give jmjatlanta some peerplay + BOOST_TEST_MESSAGE("Transferring peerplay from Nathan to jmjatlanta"); + for(int i = 1; i <= 199; i++) + { + signed_transaction transfer_tx = con.wallet_api_ptr->transfer("nathan", "jmjatlanta", std::to_string(i), + "1.3.0", "Here are some CORE token for your new account", true); + } + + BOOST_CHECK(generate_block(app1)); + + // now get account history and make sure everything is there (and no duplicates) + std::vector history = con.wallet_api_ptr->get_account_history("jmjatlanta", 300); + BOOST_CHECK_EQUAL(201u, history.size() ); + + std::set operation_ids; + + for(auto& op : history) + { + if( operation_ids.find(op.op.id) != operation_ids.end() ) + { + BOOST_FAIL("Duplicate found"); + } + operation_ids.insert(op.op.id); + } + } catch( fc::exception& e ) { + edump((e.to_detail_string())); + throw; + } +} \ No newline at end of file diff --git a/tests/common/database_fixture.cpp b/tests/common/database_fixture.cpp index a3e3a02f..f31ec00a 100644 --- a/tests/common/database_fixture.cpp +++ b/tests/common/database_fixture.cpp @@ -373,7 +373,7 @@ void database_fixture::open_database() { if( !data_dir ) { data_dir = fc::temp_directory( graphene::utilities::temp_directory_path() ); - db.open(data_dir->path(), [this]{return genesis_state;}); + db.open(data_dir->path(), [this]{return genesis_state;}, "test"); } } @@ -699,7 +699,6 @@ const account_object& database_fixture::create_account( trx.validate(); processed_transaction ptx = db.push_transaction(trx, ~0); - //wdump( (ptx) ); const account_object& result = db.get(ptx.operation_results[0].get()); trx.operations.clear(); return result; @@ -769,7 +768,6 @@ const limit_order_object*database_fixture::create_sell_order(account_id_type use const limit_order_object* database_fixture::create_sell_order( const account_object& user, const asset& amount, const asset& recv ) { - //wdump((amount)(recv)); limit_order_create_operation buy_order; buy_order.seller = user.id; buy_order.amount_to_sell = amount; @@ -780,7 +778,6 @@ const limit_order_object* database_fixture::create_sell_order( const account_obj auto processed = db.push_transaction(trx, ~0); trx.operations.clear(); verify_asset_supplies(db); - //wdump((processed)); return db.find( processed.operation_results[0].get() ); } diff --git a/tests/generate_empty_blocks/main.cpp b/tests/generate_empty_blocks/main.cpp index 1b45340d..1960a151 100644 --- a/tests/generate_empty_blocks/main.cpp +++ b/tests/generate_empty_blocks/main.cpp @@ -102,7 +102,7 @@ int main( int argc, char** argv ) std::cerr << "embed_genesis: Reading genesis from file " << genesis_json_filename.preferred_string() << "\n"; std::string genesis_json; read_file_contents( genesis_json_filename, genesis_json ); - genesis = fc::json::from_string( genesis_json ).as< genesis_state_type >(); + genesis = fc::json::from_string( genesis_json ).as< genesis_state_type >(20); } else genesis = graphene::app::detail::create_example_genesis(); @@ -119,12 +119,11 @@ int main( int argc, char** argv ) uint32_t num_blocks = options["num-blocks"].as(); uint32_t miss_rate = options["miss-rate"].as(); - fc::ecc::private_key init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); fc::ecc::private_key nathan_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("nathan"))); database db; fc::path db_path = data_dir / "db"; - db.open(db_path, [&]() { return genesis; } ); + db.open(db_path, [&]() { return genesis; }, "TEST" ); uint32_t slot = 1; uint32_t missed = 0; diff --git a/tests/tests/affiliate_tests.cpp b/tests/tests/affiliate_tests.cpp index ab109ad3..72482a0a 100644 --- a/tests/tests/affiliate_tests.cpp +++ b/tests/tests/affiliate_tests.cpp @@ -402,16 +402,16 @@ BOOST_AUTO_TEST_CASE( affiliate_payout_helper_test ) { // Fix total supply - auto& index = db.get_index_type().indices().get(); - auto itr = index.find( boost::make_tuple( account_id_type(), asset_id_type() ) ); - BOOST_CHECK( itr != index.end() ); - db.modify( *itr, [&ath]( account_balance_object& bal ) { + auto& index = db.get_index_type< primary_index< account_balance_index > >().get_secondary_index(); + auto abo = index.get_account_balance( account_id_type(), asset_id_type() ); + BOOST_CHECK( abo != nullptr ); + db.modify( *abo, [&ath]( account_balance_object& bal ) { bal.balance -= ath.alice_ppy + ath.ann_ppy + ath.audrey_ppy; }); - itr = index.find( boost::make_tuple( irene_id, btc_id ) ); - BOOST_CHECK( itr != index.end() ); - db.modify( *itr, [alice_btc,ann_btc,audrey_btc]( account_balance_object& bal ) { + abo = index.get_account_balance( irene_id, btc_id ); + BOOST_CHECK( abo != nullptr ); + db.modify( *abo, [alice_btc,ann_btc,audrey_btc]( account_balance_object& bal ) { bal.balance -= alice_btc + ann_btc + audrey_btc; }); } diff --git a/tests/tests/authority_tests.cpp b/tests/tests/authority_tests.cpp index 02d61441..2afd12a6 100644 --- a/tests/tests/authority_tests.cpp +++ b/tests/tests/authority_tests.cpp @@ -84,8 +84,7 @@ BOOST_AUTO_TEST_CASE( any_two_of_three ) trx.operations.push_back(op); sign(trx, nathan_key1); PUSH_TX( db, trx, database::skip_transaction_dupe_check ); - trx.operations.clear(); - trx.signatures.clear(); + trx.clear(); } FC_CAPTURE_AND_RETHROW ((nathan.active)) transfer_operation op; @@ -99,19 +98,19 @@ BOOST_AUTO_TEST_CASE( any_two_of_three ) PUSH_TX( db, trx, database::skip_transaction_dupe_check ); BOOST_CHECK_EQUAL(get_balance(nathan, core), static_cast(old_balance - 500)); - trx.signatures.clear(); + trx.clear_signatures(); sign(trx, nathan_key2); sign(trx, nathan_key3); PUSH_TX( db, trx, database::skip_transaction_dupe_check ); BOOST_CHECK_EQUAL(get_balance(nathan, core), static_cast(old_balance - 1000)); - trx.signatures.clear(); + trx.clear_signatures(); sign(trx, nathan_key1); sign(trx, nathan_key3); PUSH_TX( db, trx, database::skip_transaction_dupe_check ); BOOST_CHECK_EQUAL(get_balance(nathan, core), static_cast(old_balance - 1500)); - trx.signatures.clear(); + trx.clear_signatures(); //sign(trx, fc::ecc::private_key::generate()); sign(trx,nathan_key3); GRAPHENE_CHECK_THROW(PUSH_TX( db, trx, database::skip_transaction_dupe_check ), fc::exception); @@ -156,7 +155,7 @@ BOOST_AUTO_TEST_CASE( recursive_accounts ) BOOST_TEST_MESSAGE( "Attempting to transfer with parent1 signature, should fail" ); sign(trx,parent1_key); GRAPHENE_CHECK_THROW(PUSH_TX( db, trx, database::skip_transaction_dupe_check ), fc::exception); - trx.signatures.clear(); + trx.clear_signatures(); BOOST_TEST_MESSAGE( "Attempting to transfer with parent2 signature, should fail" ); sign(trx,parent2_key); @@ -166,8 +165,7 @@ BOOST_AUTO_TEST_CASE( recursive_accounts ) sign(trx,parent1_key); PUSH_TX( db, trx, database::skip_transaction_dupe_check ); BOOST_CHECK_EQUAL(get_balance(child, core), static_cast(old_balance - 500)); - trx.operations.clear(); - trx.signatures.clear(); + trx.clear(); BOOST_TEST_MESSAGE( "Adding a key for the child that can override parents" ); fc::ecc::private_key child_key = fc::ecc::private_key::generate(); @@ -181,8 +179,7 @@ BOOST_AUTO_TEST_CASE( recursive_accounts ) sign(trx,parent2_key); PUSH_TX( db, trx, database::skip_transaction_dupe_check ); BOOST_REQUIRE_EQUAL(child.active.num_auths(), 3u); - trx.operations.clear(); - trx.signatures.clear(); + trx.clear(); } op.from = child.id; @@ -195,7 +192,7 @@ BOOST_AUTO_TEST_CASE( recursive_accounts ) BOOST_TEST_MESSAGE( "Attempting transfer just parent1, should fail" ); sign(trx, parent1_key); GRAPHENE_CHECK_THROW(PUSH_TX( db, trx, database::skip_transaction_dupe_check ), fc::exception); - trx.signatures.clear(); + trx.clear_signatures(); BOOST_TEST_MESSAGE( "Attempting transfer just parent2, should fail" ); sign(trx, parent2_key); GRAPHENE_CHECK_THROW(PUSH_TX( db, trx, database::skip_transaction_dupe_check ), fc::exception); @@ -204,14 +201,13 @@ BOOST_AUTO_TEST_CASE( recursive_accounts ) sign(trx, parent1_key); PUSH_TX( db, trx, database::skip_transaction_dupe_check ); BOOST_CHECK_EQUAL(get_balance(child, core), static_cast(old_balance - 1000)); - trx.signatures.clear(); + trx.clear_signatures(); BOOST_TEST_MESSAGE( "Attempting transfer with just child key, should succeed" ); sign(trx, child_key); PUSH_TX( db, trx, database::skip_transaction_dupe_check ); BOOST_CHECK_EQUAL(get_balance(child, core), static_cast(old_balance - 1500)); - trx.operations.clear(); - trx.signatures.clear(); + trx.clear(); BOOST_TEST_MESSAGE( "Creating grandparent account, parent1 now requires authority of grandparent" ); auto grandparent = create_account("grandparent"); @@ -227,8 +223,7 @@ BOOST_AUTO_TEST_CASE( recursive_accounts ) op.owner = *op.active; trx.operations.push_back(op); PUSH_TX( db, trx, ~0 ); - trx.operations.clear(); - trx.signatures.clear(); + trx.clear(); } BOOST_TEST_MESSAGE( "Attempt to transfer using old parent keys, should fail" ); @@ -236,7 +231,7 @@ BOOST_AUTO_TEST_CASE( recursive_accounts ) sign(trx, parent1_key); sign(trx, parent2_key); GRAPHENE_CHECK_THROW(PUSH_TX( db, trx, database::skip_transaction_dupe_check ), fc::exception); - trx.signatures.clear(); + trx.clear_signatures(); sign( trx, parent2_key ); sign( trx, grandparent_key ); @@ -253,8 +248,7 @@ BOOST_AUTO_TEST_CASE( recursive_accounts ) op.owner = *op.active; trx.operations.push_back(op); PUSH_TX( db, trx, ~0 ); - trx.operations.clear(); - trx.signatures.clear(); + trx.clear(); } BOOST_TEST_MESSAGE( "Create recursion depth failure" ); @@ -265,12 +259,11 @@ BOOST_AUTO_TEST_CASE( recursive_accounts ) //Fails due to recursion depth. GRAPHENE_CHECK_THROW(PUSH_TX( db, trx, database::skip_transaction_dupe_check ), fc::exception); BOOST_TEST_MESSAGE( "verify child key can override recursion checks" ); - trx.signatures.clear(); + trx.clear_signatures(); sign(trx, child_key); PUSH_TX( db, trx, database::skip_transaction_dupe_check ); BOOST_CHECK_EQUAL(get_balance(child, core), static_cast(old_balance - 2500)); - trx.operations.clear(); - trx.signatures.clear(); + trx.clear(); BOOST_TEST_MESSAGE( "Verify a cycle fails" ); { @@ -280,8 +273,7 @@ BOOST_AUTO_TEST_CASE( recursive_accounts ) op.owner = *op.active; trx.operations.push_back(op); PUSH_TX( db, trx, ~0 ); - trx.operations.clear(); - trx.signatures.clear(); + trx.clear(); } trx.operations.push_back(op); @@ -372,7 +364,7 @@ BOOST_AUTO_TEST_CASE( proposed_single_account ) //committee has no stake in the transaction. GRAPHENE_CHECK_THROW(PUSH_TX( db, trx ), fc::exception); - trx.signatures.clear(); + trx.clear_signatures(); pup.active_approvals_to_add.clear(); pup.active_approvals_to_add.insert(nathan.id); @@ -408,7 +400,7 @@ BOOST_AUTO_TEST_CASE( proposal_failure ) pop.expiration_time = db.head_block_time() + fc::days(1); pop.fee_paying_account = bob_id; trx.operations.push_back( pop ); - trx.signatures.clear(); + trx.clear_signatures(); sign( trx, bob_private_key ); processed_transaction processed = PUSH_TX( db, trx ); proposal_object prop = db.get(processed.operation_results.front().get()); @@ -456,7 +448,7 @@ BOOST_AUTO_TEST_CASE( committee_authority ) sign(trx, committee_key); GRAPHENE_CHECK_THROW(PUSH_TX( db, trx ), graphene::chain::invalid_committee_approval ); - auto _sign = [&] { trx.signatures.clear(); sign( trx, nathan_key ); }; + auto _sign = [&] { trx.clear_signatures(); sign( trx, nathan_key ); }; proposal_create_operation pop; pop.proposed_ops.push_back({trx.operations.front()}); @@ -490,8 +482,7 @@ BOOST_AUTO_TEST_CASE( committee_authority ) BOOST_TEST_MESSAGE( "Checking that the proposal is not authorized to execute" ); BOOST_REQUIRE(!db.get(prop.id).is_authorized_to_execute(db)); - trx.operations.clear(); - trx.signatures.clear(); + trx.clear(); proposal_update_operation uop; uop.fee_paying_account = GRAPHENE_TEMP_ACCOUNT; uop.proposal = prop.id; @@ -512,7 +503,7 @@ BOOST_AUTO_TEST_CASE( committee_authority ) // fails // BOOST_CHECK(db.get(prop.id).is_authorized_to_execute(db)); - trx.signatures.clear(); + trx.clear_signatures(); generate_blocks(*prop.review_period_time); uop.key_approvals_to_add.clear(); uop.key_approvals_to_add.insert(committee_key.get_public_key()); // was 7 @@ -521,9 +512,10 @@ BOOST_AUTO_TEST_CASE( committee_authority ) // Should throw because the transaction is now in review. GRAPHENE_CHECK_THROW(PUSH_TX( db, trx ), fc::exception); - // generate_blocks(prop.expiration_time); - // fails - // BOOST_CHECK_EQUAL(get_balance(nathan, asset_id_type()(db)), 100000); + generate_blocks(prop.expiration_time); + BOOST_CHECK_EQUAL(get_balance(nathan, asset_id_type()(db)), 100000); + // proposal deleted + BOOST_CHECK_THROW( db.get(prop.id), fc::exception ); } FC_LOG_AND_RETHROW() } BOOST_FIXTURE_TEST_CASE( fired_committee_members, database_fixture ) @@ -1068,16 +1060,17 @@ BOOST_FIXTURE_TEST_CASE( bogus_signature, database_fixture ) PUSH_TX( db, trx, skip ); trx.operations.push_back( xfer_op ); + trx.signees.clear(); // signees should be invalidated BOOST_TEST_MESSAGE( "Invalidating Alices Signature" ); // Alice's signature is now invalid GRAPHENE_REQUIRE_THROW( PUSH_TX( db, trx, skip ), fc::exception ); // Re-sign, now OK (sig is replaced) BOOST_TEST_MESSAGE( "Resign with Alice's Signature" ); - trx.signatures.clear(); + trx.clear_signatures(); sign( trx, alice_key ); PUSH_TX( db, trx, skip ); - trx.signatures.clear(); + trx.clear_signatures(); trx.operations.pop_back(); sign( trx, alice_key ); sign( trx, charlie_key ); @@ -1126,7 +1119,7 @@ BOOST_FIXTURE_TEST_CASE( voting_account, database_fixture ) GRAPHENE_CHECK_THROW(PUSH_TX( db, trx ), fc::exception); op.new_options->num_committee = 3; trx.operations = {op}; - trx.signatures.clear(); + trx.clear_signatures(); sign( trx, vikram_private_key ); PUSH_TX( db, trx ); trx.clear(); diff --git a/tests/tests/basic_tests.cpp b/tests/tests/basic_tests.cpp index 834c174b..da608541 100644 --- a/tests/tests/basic_tests.cpp +++ b/tests/tests/basic_tests.cpp @@ -540,4 +540,19 @@ BOOST_AUTO_TEST_CASE( merkle_root ) BOOST_CHECK( block.calculate_merkle_root() == c(dO) ); } +/** + * Reproduces https://github.com/bitshares/bitshares-core/issues/888 and tests fix for it. + */ +BOOST_AUTO_TEST_CASE( bitasset_feed_expiration_test ) +{ + time_point_sec now = fc::time_point::now(); + + asset_bitasset_data_object o; + + o.current_feed_publication_time = now - fc::hours(1); + o.options.feed_lifetime_sec = std::numeric_limits::max() - 1; + + BOOST_CHECK( !o.feed_is_expired( now ) ); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/tests/tests/block_tests.cpp b/tests/tests/block_tests.cpp index 53e99e5f..9f74a34c 100644 --- a/tests/tests/block_tests.cpp +++ b/tests/tests/block_tests.cpp @@ -137,9 +137,10 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) // TODO: Don't generate this here auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); signed_block cutoff_block; + uint32_t last_block; { database db; - db.open(data_dir.path(), make_genesis ); + db.open(data_dir.path(), make_genesis, "TEST" ); b = db.generate_block(db.get_slot_time(1), db.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); // TODO: Change this test when we correct #406 @@ -156,6 +157,7 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) if( cutoff_height >= 200 ) { cutoff_block = *(db.fetch_block_by_number( cutoff_height )); + last_block = db.head_block_num(); break; } } @@ -163,8 +165,10 @@ BOOST_AUTO_TEST_CASE( generate_empty_blocks ) } { database db; - db.open(data_dir.path(), []{return genesis_state_type();}); - BOOST_CHECK_EQUAL( db.head_block_num(), cutoff_block.block_num() ); + db.open(data_dir.path(), []{return genesis_state_type();}, "TEST"); + BOOST_CHECK_EQUAL( db.head_block_num(), last_block ); + while( db.head_block_num() > cutoff_block.block_num() ) + db.pop_block(); b = cutoff_block; for( uint32_t i = 0; i < 200; ++i ) { @@ -188,7 +192,7 @@ BOOST_AUTO_TEST_CASE( undo_block ) fc::temp_directory data_dir( graphene::utilities::temp_directory_path() ); { database db; - db.open(data_dir.path(), make_genesis); + db.open(data_dir.path(), make_genesis, "TEST"); fc::time_point_sec now( GRAPHENE_TESTING_GENESIS_TIMESTAMP ); std::vector< time_point_sec > time_stack; @@ -237,57 +241,112 @@ BOOST_AUTO_TEST_CASE( fork_blocks ) fc::temp_directory data_dir2( graphene::utilities::temp_directory_path() ); database db1; - db1.open(data_dir1.path(), make_genesis); + db1.open(data_dir1.path(), make_genesis, "TEST"); database db2; - db2.open(data_dir2.path(), make_genesis); + db2.open(data_dir2.path(), make_genesis, "TEST"); BOOST_CHECK( db1.get_chain_id() == db2.get_chain_id() ); auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); - for( uint32_t i = 0; i < 10; ++i ) + + BOOST_TEST_MESSAGE( "Adding blocks 1 through 10" ); + for( uint32_t i = 1; i <= 10; ++i ) { auto b = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); try { PUSH_BLOCK( db2, b ); } FC_CAPTURE_AND_RETHROW( ("db2") ); } - for( uint32_t i = 10; i < 13; ++i ) + + for( uint32_t j = 0; j <= 4; j += 4 ) { - auto b = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); - } - string db1_tip = db1.head_block_id().str(); - uint32_t next_slot = 3; - for( uint32_t i = 13; i < 16; ++i ) - { - auto b = db2.generate_block(db2.get_slot_time(next_slot), db2.get_scheduled_witness(next_slot), init_account_priv_key, database::skip_nothing); - next_slot = 1; - // notify both databases of the new block. - // only db2 should switch to the new fork, db1 should not - PUSH_BLOCK( db1, b ); + // add blocks 11 through 13 to db1 only + BOOST_TEST_MESSAGE( "Adding 3 blocks to db1 only" ); + for( uint32_t i = 11 + j; i <= 13 + j; ++i ) + { + BOOST_TEST_MESSAGE( i ); + auto b = db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + } + string db1_tip = db1.head_block_id().str(); + + // add different blocks 11 through 13 to db2 only + BOOST_TEST_MESSAGE( "Add 3 different blocks to db2 only" ); + uint32_t next_slot = 3; + for( uint32_t i = 11 + j; i <= 13 + j; ++i ) + { + BOOST_TEST_MESSAGE( i ); + auto b = db2.generate_block(db2.get_slot_time(next_slot), db2.get_scheduled_witness(next_slot), init_account_priv_key, database::skip_nothing); + next_slot = 1; + // notify both databases of the new block. + // only db2 should switch to the new fork, db1 should not + PUSH_BLOCK( db1, b ); + BOOST_CHECK_EQUAL(db1.head_block_id().str(), db1_tip); + BOOST_CHECK_EQUAL(db2.head_block_id().str(), b.id().str()); + } + + //The two databases are on distinct forks now, but at the same height. + BOOST_CHECK_EQUAL(db1.head_block_num(), 13u + j); + BOOST_CHECK_EQUAL(db2.head_block_num(), 13u + j); + BOOST_CHECK( db1.head_block_id() != db2.head_block_id() ); + + //Make a block on db2, make it invalid, then + //pass it to db1 and assert that db1 doesn't switch to the new fork. + signed_block good_block; + { + auto b = db2.generate_block(db2.get_slot_time(1), db2.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); + good_block = b; + b.transactions.emplace_back(signed_transaction()); + b.transactions.back().operations.emplace_back(transfer_operation()); + b.sign( init_account_priv_key ); + BOOST_CHECK_EQUAL(b.block_num(), 14u + j); + GRAPHENE_CHECK_THROW(PUSH_BLOCK( db1, b ), fc::exception); + + // At this point, `fetch_block_by_number` will fetch block from fork_db, + // so unable to reproduce the issue which is fixed in PR #938 + // https://github.com/bitshares/bitshares-core/pull/938 + fc::optional previous_block = db1.fetch_block_by_number(1); + BOOST_CHECK ( previous_block.valid() ); + uint32_t db1_blocks = db1.head_block_num(); + for( uint32_t curr_block_num = 2; curr_block_num <= db1_blocks; ++curr_block_num ) + { + fc::optional curr_block = db1.fetch_block_by_number( curr_block_num ); + BOOST_CHECK( curr_block.valid() ); + BOOST_CHECK_EQUAL( curr_block->previous.str(), previous_block->id().str() ); + previous_block = curr_block; + } + } + BOOST_CHECK_EQUAL(db1.head_block_num(), 13u + j); BOOST_CHECK_EQUAL(db1.head_block_id().str(), db1_tip); - BOOST_CHECK_EQUAL(db2.head_block_id().str(), b.id().str()); + + if( j == 0 ) + { + // assert that db1 switches to new fork with good block + BOOST_CHECK_EQUAL(db2.head_block_num(), 14u + j); + PUSH_BLOCK( db1, good_block ); + BOOST_CHECK_EQUAL(db1.head_block_id().str(), db2.head_block_id().str()); + } } - //The two databases are on distinct forks now, but at the same height. Make a block on db2, make it invalid, then - //pass it to db1 and assert that db1 doesn't switch to the new fork. - signed_block good_block; - BOOST_CHECK_EQUAL(db1.head_block_num(), 13); - BOOST_CHECK_EQUAL(db2.head_block_num(), 13); + // generate more blocks to push the forked blocks out of fork_db + BOOST_TEST_MESSAGE( "Adding more blocks to db1, push the forked blocks out of fork_db" ); + for( uint32_t i = 1; i <= 50; ++i ) { - auto b = db2.generate_block(db2.get_slot_time(1), db2.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); - good_block = b; - b.transactions.emplace_back(signed_transaction()); - b.transactions.back().operations.emplace_back(transfer_operation()); - b.sign( init_account_priv_key ); - BOOST_CHECK_EQUAL(b.block_num(), 14); - GRAPHENE_CHECK_THROW(PUSH_BLOCK( db1, b ), fc::exception); + db1.generate_block(db1.get_slot_time(1), db1.get_scheduled_witness(1), init_account_priv_key, database::skip_nothing); } - BOOST_CHECK_EQUAL(db1.head_block_num(), 13); - BOOST_CHECK_EQUAL(db1.head_block_id().str(), db1_tip); - // assert that db1 switches to new fork with good block - BOOST_CHECK_EQUAL(db2.head_block_num(), 14); - PUSH_BLOCK( db1, good_block ); - BOOST_CHECK_EQUAL(db1.head_block_id().str(), db2.head_block_id().str()); + { + // PR #938 make sure db is in a good state https://github.com/bitshares/bitshares-core/pull/938 + BOOST_TEST_MESSAGE( "Checking whether all blocks on disk are good" ); + fc::optional previous_block = db1.fetch_block_by_number(1); + BOOST_CHECK ( previous_block.valid() ); + uint32_t db1_blocks = db1.head_block_num(); + for( uint32_t curr_block_num = 2; curr_block_num <= db1_blocks; ++curr_block_num ) + { + fc::optional curr_block = db1.fetch_block_by_number( curr_block_num ); + BOOST_CHECK( curr_block.valid() ); + BOOST_CHECK_EQUAL( curr_block->previous.str(), previous_block->id().str() ); + previous_block = curr_block; + } + } } catch (fc::exception& e) { edump((e.to_detail_string())); throw; @@ -382,7 +441,7 @@ BOOST_AUTO_TEST_CASE( undo_pending ) fc::temp_directory data_dir( graphene::utilities::temp_directory_path() ); { database db; - db.open(data_dir.path(), make_genesis); + db.open(data_dir.path(), make_genesis, "TEST"); auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); public_key_type init_account_pub_key = init_account_priv_key.get_public_key(); @@ -447,8 +506,8 @@ BOOST_AUTO_TEST_CASE( switch_forks_undo_create ) dir2( graphene::utilities::temp_directory_path() ); database db1, db2; - db1.open(dir1.path(), make_genesis); - db2.open(dir2.path(), make_genesis); + db1.open(dir1.path(), make_genesis, "TEST"); + db2.open(dir2.path(), make_genesis, "TEST"); BOOST_CHECK( db1.get_chain_id() == db2.get_chain_id() ); auto init_account_priv_key = fc::ecc::private_key::regenerate(fc::sha256::hash(string("null_key")) ); @@ -506,8 +565,8 @@ BOOST_AUTO_TEST_CASE( duplicate_transactions ) dir2( graphene::utilities::temp_directory_path() ); database db1, db2; - db1.open(dir1.path(), make_genesis); - db2.open(dir2.path(), make_genesis); + db1.open(dir1.path(), make_genesis, "TEST"); + db2.open(dir2.path(), make_genesis, "TEST"); BOOST_CHECK( db1.get_chain_id() == db2.get_chain_id() ); auto skip_sigs = database::skip_transaction_signatures | database::skip_authority_check; @@ -556,7 +615,7 @@ BOOST_AUTO_TEST_CASE( tapos ) try { fc::temp_directory dir1( graphene::utilities::temp_directory_path() ); database db1; - db1.open(dir1.path(), make_genesis); + db1.open(dir1.path(), make_genesis, "TEST"); const account_object& init1 = *db1.get_index_type().indices().get().find("init1"); @@ -591,7 +650,7 @@ BOOST_AUTO_TEST_CASE( tapos ) //relative_expiration is 1, but ref block is 2 blocks old, so this should fail. GRAPHENE_REQUIRE_THROW(PUSH_TX( db1, trx, database::skip_transaction_signatures | database::skip_authority_check ), fc::exception); set_expiration( db1, trx ); - trx.signatures.clear(); + trx.clear_signatures(); trx.sign( init_account_priv_key, db1.get_chain_id() ); db1.push_transaction(trx, database::skip_transaction_signatures | database::skip_authority_check); } catch (fc::exception& e) { @@ -623,14 +682,14 @@ BOOST_FIXTURE_TEST_CASE( optional_tapos, database_fixture ) tx.ref_block_num = 0; tx.ref_block_prefix = 0; - tx.signatures.clear(); + tx.clear_signatures(); sign( tx, alice_private_key ); PUSH_TX( db, tx ); BOOST_TEST_MESSAGE( "proper ref_block_num, ref_block_prefix" ); set_expiration( db, tx ); - tx.signatures.clear(); + tx.clear_signatures(); sign( tx, alice_private_key ); PUSH_TX( db, tx ); @@ -638,7 +697,7 @@ BOOST_FIXTURE_TEST_CASE( optional_tapos, database_fixture ) tx.ref_block_num = 0; tx.ref_block_prefix = 0x12345678; - tx.signatures.clear(); + tx.clear_signatures(); sign( tx, alice_private_key ); GRAPHENE_REQUIRE_THROW( PUSH_TX( db, tx ), fc::exception ); @@ -646,7 +705,7 @@ BOOST_FIXTURE_TEST_CASE( optional_tapos, database_fixture ) tx.ref_block_num = 1; tx.ref_block_prefix = 0x12345678; - tx.signatures.clear(); + tx.clear_signatures(); sign( tx, alice_private_key ); GRAPHENE_REQUIRE_THROW( PUSH_TX( db, tx ), fc::exception ); @@ -654,7 +713,7 @@ BOOST_FIXTURE_TEST_CASE( optional_tapos, database_fixture ) tx.ref_block_num = 9999; tx.ref_block_prefix = 0x12345678; - tx.signatures.clear(); + tx.clear_signatures(); sign( tx, alice_private_key ); GRAPHENE_REQUIRE_THROW( PUSH_TX( db, tx ), fc::exception ); } @@ -799,8 +858,8 @@ BOOST_FIXTURE_TEST_CASE( double_sign_check, database_fixture ) BOOST_TEST_MESSAGE( "Verify that signing once with the proper key passes" ); trx.signatures.pop_back(); + trx.signees.clear(); // signees should be invalidated db.push_transaction(trx, 0); - sign( trx, bob_private_key ); } FC_LOG_AND_RETHROW() } @@ -1107,7 +1166,7 @@ BOOST_FIXTURE_TEST_CASE( transaction_invalidated_in_cache, database_fixture ) fc::temp_directory data_dir2( graphene::utilities::temp_directory_path() ); database db2; - db2.open(data_dir2.path(), make_genesis); + db2.open(data_dir2.path(), make_genesis, "TEST"); BOOST_CHECK( db.get_chain_id() == db2.get_chain_id() ); while( db2.head_block_num() < db.head_block_num() ) @@ -1159,7 +1218,7 @@ BOOST_FIXTURE_TEST_CASE( transaction_invalidated_in_cache, database_fixture ) signed_transaction tx = generate_xfer_tx( alice_id, bob_id, 1000, 2 ); tx.set_expiration( db.head_block_time() + 2 * db.get_global_properties().parameters.block_interval ); - tx.signatures.clear(); + tx.clear_signatures(); sign( tx, alice_private_key ); // put the tx in db tx cache PUSH_TX( db, tx ); @@ -1270,7 +1329,7 @@ BOOST_AUTO_TEST_CASE( genesis_reserve_ids ) genesis_state.initial_assets.push_back( usd ); return genesis_state; - } ); + }, "TEST" ); const auto& acct_idx = db.get_index_type().indices().get(); auto acct_itr = acct_idx.find("init0"); diff --git a/tests/tests/confidential_tests.cpp b/tests/tests/confidential_tests.cpp index 3f47b698..a6a19f06 100644 --- a/tests/tests/confidential_tests.cpp +++ b/tests/tests/confidential_tests.cpp @@ -80,7 +80,7 @@ BOOST_AUTO_TEST_CASE( confidential_test ) trx.operations = {to_blind}; sign( trx, dan_private_key ); db.push_transaction(trx); - trx.signatures.clear(); + trx.clear_signatures(); BOOST_TEST_MESSAGE( "Transfering from blind to blind with change address" ); auto Out3B = fc::sha256::hash("Out3B"); @@ -123,7 +123,7 @@ BOOST_AUTO_TEST_CASE( confidential_test ) from_blind.blinding_factor = Out4B; from_blind.inputs.push_back( {out4.commitment, out4.owner} ); trx.operations = {from_blind}; - trx.signatures.clear(); + trx.clear_signatures(); db.push_transaction(trx); BOOST_REQUIRE_EQUAL( get_balance( nathan, core ), 750-300-10-10 ); diff --git a/tests/tests/database_tests.cpp b/tests/tests/database_tests.cpp index 18ea8e40..9585d4a1 100644 --- a/tests/tests/database_tests.cpp +++ b/tests/tests/database_tests.cpp @@ -62,6 +62,25 @@ BOOST_AUTO_TEST_CASE( undo_test ) } } +BOOST_AUTO_TEST_CASE( merge_test ) +{ + try { + database db; + auto ses = db._undo_db.start_undo_session(); + const auto& bal_obj1 = db.create( [&]( account_balance_object& obj ){ + obj.balance = 42; + }); + ses.merge(); + + auto balance = db.get_balance( account_id_type(), asset_id_type() ); + BOOST_CHECK_EQUAL( 42, balance.amount.value ); + } catch ( const fc::exception& e ) + { + edump( (e.to_detail_string()) ); + throw; + } +} + BOOST_AUTO_TEST_CASE( flat_index_test ) { ACTORS((sam)); @@ -91,4 +110,89 @@ BOOST_AUTO_TEST_CASE( flat_index_test ) FC_ASSERT( !(*bitusd.bitasset_data_id)(db).current_feed.settlement_price.is_null() ); } +BOOST_AUTO_TEST_CASE( direct_index_test ) +{ try { + try { + const graphene::db::primary_index< account_index, 6 > small_chunkbits( db ); + BOOST_FAIL( "Expected assertion failure!" ); + } catch( const fc::assert_exception& expected ) {} + + graphene::db::primary_index< account_index, 8 > my_accounts( db ); + const auto& direct = my_accounts.get_secondary_index>(); + BOOST_CHECK_EQUAL( 0, my_accounts.indices().size() ); + BOOST_CHECK( nullptr == direct.find( account_id_type( 1 ) ) ); + // BOOST_CHECK_THROW( direct.find( asset_id_type( 1 ) ), fc::assert_exception ); // compile-time error + BOOST_CHECK_THROW( direct.find( object_id_type( asset_id_type( 1 ) ) ), fc::assert_exception ); + BOOST_CHECK_THROW( direct.get( account_id_type( 1 ) ), fc::assert_exception ); + + account_object test_account; + test_account.id = account_id_type(1); + test_account.name = "account1"; + + my_accounts.load( fc::raw::pack( test_account ) ); + + BOOST_CHECK_EQUAL( 1, my_accounts.indices().size() ); + BOOST_CHECK( nullptr == direct.find( account_id_type( 0 ) ) ); + BOOST_CHECK( nullptr == direct.find( account_id_type( 2 ) ) ); + BOOST_CHECK( nullptr != direct.find( account_id_type( 1 ) ) ); + BOOST_CHECK_EQUAL( test_account.name, direct.get( test_account.id ).name ); + + // The following assumes that MAX_HOLE = 100 + test_account.id = account_id_type(102); + test_account.name = "account102"; + // highest insert was 1, direct.next is 2 => 102 is highest allowed instance + my_accounts.load( fc::raw::pack( test_account ) ); + BOOST_CHECK_EQUAL( test_account.name, direct.get( test_account.id ).name ); + + // direct.next is now 103, but index sequence counter is 0 + my_accounts.create( [] ( object& o ) { + account_object& acct = dynamic_cast< account_object& >( o ); + BOOST_CHECK_EQUAL( 0, acct.id.instance() ); + acct.name = "account0"; + } ); + + test_account.id = account_id_type(50); + test_account.name = "account50"; + my_accounts.load( fc::raw::pack( test_account ) ); + + // can handle nested modification + my_accounts.modify( direct.get( account_id_type(0) ), [&direct,&my_accounts] ( object& outer ) { + account_object& _outer = dynamic_cast< account_object& >( outer ); + my_accounts.modify( direct.get( account_id_type(50) ), [] ( object& inner ) { + account_object& _inner = dynamic_cast< account_object& >( inner ); + _inner.referrer = account_id_type(102); + }); + _outer.options.voting_account = GRAPHENE_PROXY_TO_SELF_ACCOUNT; + }); + + // direct.next is still 103, so 204 is not allowed + test_account.id = account_id_type(204); + test_account.name = "account204"; + GRAPHENE_REQUIRE_THROW( my_accounts.load( fc::raw::pack( test_account ) ), fc::assert_exception ); + // This is actually undefined behaviour. The object has been inserted into + // the primary index, but the secondary has refused to insert it! + BOOST_CHECK_EQUAL( 5, my_accounts.indices().size() ); + + uint32_t count = 0; + for( uint32_t i = 0; i < 250; i++ ) + { + const account_object* aptr = dynamic_cast< const account_object* >( my_accounts.find( account_id_type( i ) ) ); + if( aptr ) + { + count++; + BOOST_CHECK( aptr->id.instance() == 0 || aptr->id.instance() == 1 + || aptr->id.instance() == 50 || aptr->id.instance() == 102 ); + BOOST_CHECK_EQUAL( i, aptr->id.instance() ); + BOOST_CHECK_EQUAL( "account" + std::to_string( i ), aptr->name ); + } + } + BOOST_CHECK_EQUAL( count, my_accounts.indices().size() - 1 ); + + GRAPHENE_REQUIRE_THROW( my_accounts.modify( direct.get( account_id_type( 1 ) ), [] ( object& acct ) { + acct.id = account_id_type(2); + }), fc::assert_exception ); + // This is actually undefined behaviour. The object has been modified, but + // but the secondary has not updated its representation +} FC_LOG_AND_RETHROW() } + BOOST_AUTO_TEST_SUITE_END() diff --git a/tests/tests/network_broadcast_api_tests.cpp b/tests/tests/network_broadcast_api_tests.cpp index 48165489..4c3e7ed4 100644 --- a/tests/tests/network_broadcast_api_tests.cpp +++ b/tests/tests/network_broadcast_api_tests.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "../common/database_fixture.hpp" @@ -398,6 +399,8 @@ BOOST_AUTO_TEST_CASE( check_passes_for_duplicated_betting_market_or_group ) proposal_create_operation pcop2 = pcop1; + trx.clear(); + pcop1.proposed_ops.emplace_back( evcop1 ); pcop1.proposed_ops.emplace_back( bmgcop ); pcop1.proposed_ops.emplace_back( bmcop ); @@ -419,3 +422,44 @@ BOOST_AUTO_TEST_CASE( check_passes_for_duplicated_betting_market_or_group ) } BOOST_AUTO_TEST_SUITE_END() + +BOOST_FIXTURE_TEST_SUITE(network_broadcast_api_tests, database_fixture) + +BOOST_AUTO_TEST_CASE( broadcast_transaction_with_callback_test ) { + try { + + uint32_t called = 0; + auto callback = [&]( const variant& v ) + { + ++called; + }; + + fc::ecc::private_key cid_key = fc::ecc::private_key::regenerate( fc::digest("key") ); + const account_id_type cid_id = create_account( "cid", cid_key.get_public_key() ).id; + fund( cid_id(db) ); + + auto nb_api = std::make_shared< graphene::app::network_broadcast_api >( app ); + + set_expiration( db, trx ); + transfer_operation trans; + trans.from = cid_id; + trans.to = account_id_type(); + trans.amount = asset(1); + trx.operations.push_back( trans ); + sign( trx, cid_key ); + + nb_api->broadcast_transaction_with_callback( callback, trx ); + + trx.operations.clear(); + trx.signatures.clear(); + + generate_block(); + + fc::usleep(fc::milliseconds(200)); // sleep a while to execute callback in another thread + + BOOST_CHECK_EQUAL( called, 1 ); + + } FC_LOG_AND_RETHROW() +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/tests/tests/network_node_api_tests.cpp b/tests/tests/network_node_api_tests.cpp index b857cdfe..88b762ea 100644 --- a/tests/tests/network_node_api_tests.cpp +++ b/tests/tests/network_node_api_tests.cpp @@ -171,7 +171,7 @@ BOOST_AUTO_TEST_CASE(subscribe_to_pending_transactions) { signed_transaction transaction_in_notification; network_node_api.subscribe_to_pending_transactions([&]( const variant& signed_transaction_object ){ - transaction_in_notification = signed_transaction_object.as(); + transaction_in_notification = signed_transaction_object.as( GRAPHENE_MAX_NESTED_OBJECTS ); }); auto sam_transaction = push_transaction_for_account_creation("sam"); diff --git a/tests/tests/operation_tests2.cpp b/tests/tests/operation_tests2.cpp index 98d40207..b68b34a7 100644 --- a/tests/tests/operation_tests2.cpp +++ b/tests/tests/operation_tests2.cpp @@ -675,7 +675,7 @@ BOOST_AUTO_TEST_CASE( worker_pay_test ) trx.operations.push_back(op); sign( trx, nathan_private_key ); PUSH_TX( db, trx ); - trx.signatures.clear(); + trx.clear_signatures(); REQUIRE_THROW_WITH_VALUE(op, amount, asset(1)); trx.clear(); } @@ -710,7 +710,7 @@ BOOST_AUTO_TEST_CASE( worker_pay_test ) trx.operations.back() = op; sign( trx, nathan_private_key ); PUSH_TX( db, trx ); - trx.signatures.clear(); + trx.clear_signatures(); trx.clear(); } @@ -1111,7 +1111,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test ) auto _sign = [&]( signed_transaction& tx, const private_key_type& key ) { tx.sign( key, db.get_chain_id() ); }; - db.open(td.path(), [this]{return genesis_state;}); + db.open(td.path(), [this]{return genesis_state;}, "TEST"); const balance_object& balance = balance_id_type()(db); BOOST_CHECK_EQUAL(balance.balance.amount.value, 1); BOOST_CHECK_EQUAL(balance_id_type(1)(db).balance.amount.value, 1); @@ -1164,7 +1164,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test ) op.total_claimed.amount = 151; op.balance_owner_key = v2_key.get_public_key(); trx.operations = {op}; - trx.signatures.clear(); + trx.clear_signatures(); _sign( trx, n_key ); _sign( trx, v2_key ); // Attempting to claim 151 from a balance with 150 available @@ -1174,7 +1174,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test ) op.total_claimed.amount = 100; op.balance_owner_key = v2_key.get_public_key(); trx.operations = {op}; - trx.signatures.clear(); + trx.clear_signatures(); _sign( trx, n_key ); _sign( trx, v2_key ); db.push_transaction(trx); @@ -1183,7 +1183,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test ) op.total_claimed.amount = 10; trx.operations = {op}; - trx.signatures.clear(); + trx.clear_signatures(); _sign( trx, n_key ); _sign( trx, v2_key ); // Attempting to claim twice within a day @@ -1198,7 +1198,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test ) op.total_claimed.amount = 500; op.balance_owner_key = v1_key.get_public_key(); trx.operations = {op}; - trx.signatures.clear(); + trx.clear_signatures(); _sign( trx, n_key ); _sign( trx, v1_key ); db.push_transaction(trx); @@ -1209,7 +1209,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test ) op.balance_owner_key = v2_key.get_public_key(); op.total_claimed.amount = 10; trx.operations = {op}; - trx.signatures.clear(); + trx.clear_signatures(); _sign( trx, n_key ); _sign( trx, v2_key ); // Attempting to claim twice within a day @@ -1222,7 +1222,7 @@ BOOST_AUTO_TEST_CASE( balance_object_test ) op.total_claimed = vesting_balance_2.balance; trx.operations = {op}; - trx.signatures.clear(); + trx.clear_signatures(); _sign( trx, n_key ); _sign( trx, v2_key ); db.push_transaction(trx); @@ -1650,7 +1650,7 @@ BOOST_AUTO_TEST_CASE( buyback ) // Alice and Philbin signed, but asset issuer is invalid GRAPHENE_CHECK_THROW( db.push_transaction(tx), account_create_buyback_incorrect_issuer ); - tx.signatures.clear(); + tx.clear_signatures(); tx.operations.back().get< account_create_operation >().extensions.value.buyback_options->asset_to_buy_issuer = izzy_id; sign( tx, philbin_private_key ); @@ -1663,7 +1663,7 @@ BOOST_AUTO_TEST_CASE( buyback ) rex_id = ptx.operation_results.back().get< object_id_type >(); // Try to create another account rex2 which is bbo on same asset - tx.signatures.clear(); + tx.clear_signatures(); tx.operations.back().get< account_create_operation >().name = "rex2"; sign( tx, izzy_private_key ); sign( tx, philbin_private_key ); diff --git a/tests/tests/serialization_tests.cpp b/tests/tests/serialization_tests.cpp index fb87c4c4..59e16f01 100644 --- a/tests/tests/serialization_tests.cpp +++ b/tests/tests/serialization_tests.cpp @@ -64,8 +64,8 @@ BOOST_AUTO_TEST_CASE( serialization_json_test ) op.to = account_id_type(2); op.amount = asset(100); trx.operations.push_back( op ); - fc::variant packed(trx); - signed_transaction unpacked = packed.as(); + fc::variant packed(trx, GRAPHENE_MAX_NESTED_OBJECTS); + signed_transaction unpacked = packed.as( GRAPHENE_MAX_NESTED_OBJECTS ); unpacked.validate(); BOOST_CHECK( digest(trx) == digest(unpacked) ); } catch (fc::exception& e) { diff --git a/tests/tests/uia_tests.cpp b/tests/tests/uia_tests.cpp index f1d6bb57..78cd0f82 100644 --- a/tests/tests/uia_tests.cpp +++ b/tests/tests/uia_tests.cpp @@ -99,7 +99,7 @@ BOOST_AUTO_TEST_CASE( override_transfer_test ) sign( trx, dan_private_key ); GRAPHENE_REQUIRE_THROW( PUSH_TX( db, trx, 0 ), tx_missing_active_auth ); BOOST_TEST_MESSAGE( "Pass with issuer's signature" ); - trx.signatures.clear(); + trx.clear_signatures(); sign( trx, sam_private_key ); PUSH_TX( db, trx, 0 ); @@ -128,7 +128,7 @@ BOOST_AUTO_TEST_CASE( override_transfer_test2 ) sign( trx, dan_private_key ); GRAPHENE_REQUIRE_THROW( PUSH_TX( db, trx, 0 ), fc::exception); BOOST_TEST_MESSAGE( "Fail because overide_authority flag is not set" ); - trx.signatures.clear(); + trx.clear_signatures(); sign( trx, sam_private_key ); GRAPHENE_REQUIRE_THROW( PUSH_TX( db, trx, 0 ), fc::exception );